cmd/compile: export inlined function bodies

Completed implementation for exporting inlined functions
using the new binary export format. This change passes
(export GO_GCFLAGS=-newexport; make all.bash) but for
gc's builtin_test.go which we need to adjust before enabling
this code by default.

For a high-level description of the export format see the
comment at the top of bexport.go.

Major changes:

1) The export format for the platform independent export data
   changed: When we export inlined function bodies, additional
   objects (other functions, types, etc.) that are referred to
   by the function bodies will need to be exported. While this
   doesn't affect the platform-independent portion directly, it
   adds more objects to the exportlist while we are exporting.
   Instead of trying to sort the objects into groups, just export
   objects as they appear in the export list. This is slightly
   less compact (one extra byte per object), but it is simpler
   and much more flexible.

2) The export format contains now three sections: 1) The plat-
   form independent objects, 2) the objects pulled in for export
   via inlined function bodies, and 3) the inlined function bodies.

3) Completed the exporting and importing code for inlined function
   bodies. The format is completely compiler-specific and easily
   changeable w/o affecting other tools. There is still quite a
   bit of room for denser encoding. This can happen at any time
   in the future.

This change contains also the adjustments for go/internal/gcimporter,
necessary because of the export format change 1) mentioned above.

For #13241.

Change-Id: I86bca0bd984b12ccf13d0d30892e6e25f6d04ed5
Reviewed-on: https://go-review.googlesource.com/21172
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/compile/internal/gc/bimport.go

@@ -19,6 +19,19 @@ import (
 // in bimport.go. Changing the export format requires making symmetric
 // changes to bimport.go and bexport.go.
 
+type importer struct {
+	in       *bufio.Reader
+	buf      []byte   // for reading strings
+	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
+	pkgList  []*Pkg
+	typList  []*Type
+	inlined  []*Node // functions with pending inlined function bodies
+
+	// debugging support
+	debugFormat bool
+	read        int // bytes read
+}
+
 // Import populates importpkg from the serialized package data.
 func Import(in *bufio.Reader) {
 	p := importer{in: in}
@@ -58,68 +71,70 @@ func Import(in *bufio.Reader) {
 	typecheckok = true
 	defercheckwidth()
 
-	// read consts
+	// read objects
-	for i := p.int(); i > 0; i-- {
-		sym := p.localname()
-		typ := p.typ()
-		val := p.value(typ)
-		importconst(sym, idealType(typ), nodlit(val))
-	}
 
-	// read vars
+	// Phase 1
-	for i := p.int(); i > 0; i-- {
+	objcount := 0
-		sym := p.localname()
+	for {
-		typ := p.typ()
+		tag := p.tagOrIndex()
-		importvar(sym, typ)
+		if tag == endTag {
-	}
+			break
-
-	// read funcs
-	for i := p.int(); i > 0; i-- {
-		// parser.go:hidden_fndcl
-		sym := p.localname()
-		params := p.paramList()
-		result := p.paramList()
-		inl := p.int()
-
-		sig := functype(nil, params, result)
-		importsym(sym, ONAME)
-		if sym.Def != nil && sym.Def.Op == ONAME && !Eqtype(sig, sym.Def.Type) {
-			Fatalf("importer: inconsistent definition for func %v during import\n\t%v\n\t%v", sym, sym.Def.Type, sig)
 		}
-
+		p.obj(tag)
-		n := newfuncname(sym)
+		objcount++
-		n.Type = sig
-		declare(n, PFUNC)
-		funchdr(n)
-
-		// parser.go:hidden_import
-		n.Func.Inl.Set(nil)
-		if inl >= 0 {
-			if inl != len(p.inlined) {
-				panic(fmt.Sprintf("inlined body list inconsistent: %d != %d", inl, len(p.inlined)))
-			}
-			p.inlined = append(p.inlined, n.Func)
-		}
-		funcbody(n)
-		importlist = append(importlist, n) // TODO(gri) do this only if body is inlineable?
 	}
 
-	// read types
+	// self-verification
-	for i := p.int(); i > 0; i-- {
+	if count := p.int(); count != objcount {
-		// name is parsed as part of named type
+		Fatalf("importer: got %d objects; want %d", objcount, count)
-		p.typ()
 	}
 
 	// --- compiler-specific export data ---
 
-	// read inlined functions bodies
+	// Phase 2
-	n := p.int()
+	objcount = 0
-	for i := 0; i < n; i++ {
+	for {
-		body := p.block()
+		tag := p.tagOrIndex()
-		const hookup = false // TODO(gri) enable and remove this condition
+		if tag == endTag {
-		if hookup {
+			break
-			p.inlined[i].Inl.Set(body)
 		}
+		p.obj(tag)
+		objcount++
+	}
+
+	// self-verification
+	if count := p.int(); count != objcount {
+		Fatalf("importer: got %d objects; want %d", objcount, count)
+	}
+
+	// read inlined functions bodies
+	if dclcontext != PEXTERN {
+		Fatalf("importer: unexpected context %d", dclcontext)
+	}
+
+	bcount := p.int() // consistency check only
+	if bcount != len(p.inlined) {
+		Fatalf("importer: expected %d inlined function bodies; got %d", bcount, len(p.inlined))
+	}
+	for _, f := range p.inlined {
+		if Funcdepth != 0 {
+			Fatalf("importer: unexpected Funcdepth %d", Funcdepth)
+		}
+		if f != nil {
+			// function body not yet imported - read body and set it
+			funchdr(f)
+			f.Func.Inl.Set(p.stmtList())
+			funcbody(f)
+		} else {
+			// function already imported - read body but discard declarations
+			dclcontext = PDISCARD // throw away any declarations
+			p.stmtList()
+			dclcontext = PEXTERN
+		}
+	}
+
+	if dclcontext != PEXTERN {
+		Fatalf("importer: unexpected context %d", dclcontext)
 	}
 
 	// --- end of export data ---
@@ -130,26 +145,6 @@ func Import(in *bufio.Reader) {
 	testdclstack() // debugging only
 }
 
-func idealType(typ *Type) *Type {
-	if typ.IsUntyped() {
-		// canonicalize ideal types
-		typ = Types[TIDEAL]
-	}
-	return typ
-}
-
-type importer struct {
-	in       *bufio.Reader
-	buf      []byte   // for reading strings
-	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
-	pkgList  []*Pkg
-	typList  []*Type
-	inlined  []*Func
-
-	debugFormat bool
-	read        int // bytes read
-}
-
 func (p *importer) pkg() *Pkg {
 	// if the package was seen before, i is its index (>= 0)
 	i := p.tagOrIndex()
@@ -184,20 +179,93 @@ func (p *importer) pkg() *Pkg {
 	if pkg.Name == "" {
 		pkg.Name = name
 	} else if pkg.Name != name {
-		Fatalf("importer: inconsistent package names: got %s; want %s (path = %s)", pkg.Name, name, path)
+		Fatalf("importer: conflicting names %s and %s for package %q", pkg.Name, name, path)
 	}
 	p.pkgList = append(p.pkgList, pkg)
 
 	return pkg
 }
 
-func (p *importer) localname() *Sym {
+func idealType(typ *Type) *Type {
-	// parser.go:hidden_importsym
+	if typ.IsUntyped() {
-	name := p.string()
+		// canonicalize ideal types
-	if name == "" {
+		typ = Types[TIDEAL]
-		Fatalf("importer: unexpected anonymous name")
+	}
+	return typ
+}
+
+func (p *importer) obj(tag int) {
+	switch tag {
+	case constTag:
+		sym := p.qualifiedName()
+		typ := p.typ()
+		val := p.value(typ)
+		importconst(sym, idealType(typ), nodlit(val))
+
+	case typeTag:
+		p.typ()
+
+	case varTag:
+		sym := p.qualifiedName()
+		typ := p.typ()
+		importvar(sym, typ)
+
+	case funcTag:
+		sym := p.qualifiedName()
+		params := p.paramList()
+		result := p.paramList()
+		inl := p.int()
+
+		sig := functype(nil, params, result)
+		importsym(sym, ONAME)
+		if sym.Def != nil && sym.Def.Op == ONAME {
+			if Eqtype(sig, sym.Def.Type) {
+				// function was imported before (via another import)
+				dclcontext = PDISCARD // since we skip funchdr below
+			} else {
+				Fatalf("importer: inconsistent definition for func %v during import\n\t%v\n\t%v", sym, sym.Def.Type, sig)
+			}
+		}
+
+		var n *Node
+		if dclcontext != PDISCARD {
+			n = newfuncname(sym)
+			n.Type = sig
+			declare(n, PFUNC)
+			if inl < 0 {
+				funchdr(n)
+			}
+		}
+
+		if inl >= 0 {
+			// function has inlined body - collect for later
+			if inl != len(p.inlined) {
+				Fatalf("importer: inlined index = %d; want %d", inl, len(p.inlined))
+			}
+			p.inlined = append(p.inlined, n)
+		}
+
+		// parser.go:hidden_import
+		if dclcontext == PDISCARD {
+			dclcontext = PEXTERN // since we skip the funcbody below
+			break
+		}
+
+		if inl < 0 {
+			funcbody(n)
+		}
+		importlist = append(importlist, n) // TODO(gri) may only be needed for inlineable functions
+
+		if Debug['E'] > 0 {
+			fmt.Printf("import [%q] func %v \n", importpkg.Path, n)
+			if Debug['m'] > 2 && len(n.Func.Inl.Slice()) != 0 {
+				fmt.Printf("inl body: %v\n", n.Func.Inl)
+			}
+		}
+
+	default:
+		Fatalf("importer: unexpected object tag")
 	}
-	return importpkg.Lookup(name)
 }
 
 func (p *importer) newtyp(etype EType) *Type {
@@ -235,26 +303,37 @@ func (p *importer) typ() *Type {
 			break
 		}
 
+		// set correct import context (since p.typ() may be called
+		// while importing the body of an inlined function)
+		savedContext := dclcontext
+		dclcontext = PEXTERN
+
 		// read associated methods
 		for i := p.int(); i > 0; i-- {
 			// parser.go:hidden_fndcl
-			name := p.string()
+
+			sym := p.fieldSym()
+
 			recv := p.paramList() // TODO(gri) do we need a full param list for the receiver?
 			params := p.paramList()
 			result := p.paramList()
 			inl := p.int()
 
-			pkg := localpkg
-			if !exportname(name) {
-				pkg = tsym.Pkg
-			}
-			sym := pkg.Lookup(name)
-
 			n := methodname1(newname(sym), recv[0].Right)
 			n.Type = functype(recv[0], params, result)
 			checkwidth(n.Type)
 			addmethod(sym, n.Type, tsym.Pkg, false, false)
-			funchdr(n)
+			if inl < 0 {
+				funchdr(n)
+			}
+
+			if inl >= 0 {
+				// method has inlined body - collect for later
+				if inl != len(p.inlined) {
+					Fatalf("importer: inlined index = %d; want %d", inl, len(p.inlined))
+				}
+				p.inlined = append(p.inlined, n)
+			}
 
 			// (comment from parser.go)
 			// inl.C's inlnode in on a dotmeth node expects to find the inlineable body as
@@ -264,17 +343,21 @@ func (p *importer) typ() *Type {
 			n.Type.SetNname(n)
 
 			// parser.go:hidden_import
-			n.Func.Inl.Set(nil)
+			if inl < 0 {
-			if inl >= 0 {
+				funcbody(n)
-				if inl != len(p.inlined) {
+			}
-					panic(fmt.Sprintf("inlined body list inconsistent: %d != %d", inl, len(p.inlined)))
+			importlist = append(importlist, n) // TODO(gri) may only be needed for inlineable functions
-				}
+
-				p.inlined = append(p.inlined, n.Func)
+			if Debug['E'] > 0 {
+				fmt.Printf("import [%q] meth %v \n", importpkg.Path, n)
+				if Debug['m'] > 2 && len(n.Func.Inl.Slice()) != 0 {
+					fmt.Printf("inl body: %v\n", n.Func.Inl)
+				}
 			}
-			funcbody(n)
-			importlist = append(importlist, n) // TODO(gri) do this only if body is inlineable?
 		}
 
+		dclcontext = savedContext
+
 	case arrayTag, sliceTag:
 		t = p.newtyp(TARRAY)
 		if i == arrayTag {
@@ -412,7 +495,7 @@ func (p *importer) fieldName() *Sym {
 		// During imports, unqualified non-exported identifiers are from builtinpkg
 		// (see parser.go:sym). The binary exporter only exports blank as a non-exported
 		// identifier without qualification.
-		pkg = builtinpkg
+		pkg = localpkg
 	} else if name == "?" || name != "" && !exportname(name) {
 		if name == "?" {
 			name = ""
@@ -461,9 +544,10 @@ func (p *importer) param(named bool) *Node {
 		if name == "" {
 			Fatalf("importer: expected named parameter")
 		}
-		// The parameter package doesn't matter; it's never consulted.
+		// TODO(gri) Supply function/method package rather than
-		// We use the builtinpkg per parser.go:sym (line 1181).
+		// encoding the package for each parameter repeatedly.
-		n.Left = newname(builtinpkg.Lookup(name))
+		pkg := p.pkg()
+		n.Left = newname(pkg.Lookup(name))
 	}
 
 	// TODO(gri) This is compiler-specific (escape info).
@@ -546,112 +630,152 @@ func (p *importer) float(x *Mpflt) {
 // ----------------------------------------------------------------------------
 // Inlined function bodies
 
-func (p *importer) block() []*Node {
+// Approach: Read nodes and use them to create/declare the same data structures
-	markdcl()
+// as done originally by the (hidden) parser by closely following the parser's
-	// TODO(gri) populate "scope" with function parameters so they can be found
+// original code. In other words, "parsing" the import data (which happens to
-	//           inside the function body
+// be encoded in binary rather textual form) is the best way at the moment to
-	list := p.nodeList()
+// re-establish the syntax tree's invariants. At some future point we might be
-	popdcl()
+// able to avoid this round-about way and create the rewritten nodes directly,
+// possibly avoiding a lot of duplicate work (name resolution, type checking).
+
+func (p *importer) stmtList() []*Node {
+	var list []*Node
+	for {
+		n := p.node()
+		if n == nil {
+			break
+		}
+		// OBLOCK nodes may be created when importing ODCL nodes - unpack them
+		if n.Op == OBLOCK {
+			list = append(list, n.List.Slice()...)
+		} else {
+			list = append(list, n)
+		}
+	}
 	return list
 }
 
-// parser.go:stmt_list
+func (p *importer) exprList() []*Node {
-func (p *importer) nodeList() []*Node {
+	var list []*Node
-	c := p.int()
+	for {
-	s := make([]*Node, c)
+		n := p.expr()
-	for i := range s {
+		if n == nil {
-		s[i] = p.node()
+			break
+		}
+		list = append(list, n)
 	}
-	return s
+	return list
 }
 
+func (p *importer) elemList() []*Node {
+	c := p.int()
+	list := make([]*Node, c)
+	for i := range list {
+		list[i] = Nod(OKEY, mkname(p.fieldSym()), p.expr())
+	}
+	return list
+}
+
+func (p *importer) expr() *Node {
+	n := p.node()
+	if n != nil && n.Op == OBLOCK {
+		Fatalf("unexpected block node: %v", n)
+	}
+	return n
+}
+
+// TODO(gri) split into expr and stmt
 func (p *importer) node() *Node {
-	// TODO(gri) eventually we may need to allocate in each branch
+	switch op := p.op(); op {
-	n := Nod(p.op(), nil, nil)
+	// expressions
+	// case OPAREN:
+	// 	unreachable - unpacked by exporter
 
-	switch n.Op {
+	// case ODDDARG:
-	// names
+	//	unimplemented
-	case ONAME, OPACK, ONONAME:
-		name := mkname(p.sym())
-		// TODO(gri) decide what to do here (this code throws away n)
-		/*
-			if name.Op != n.Op {
-				Fatalf("importer: got node op = %s; want %s", opnames[name.Op], opnames[n.Op])
-			}
-		*/
-		n = name
 
-	case OTYPE:
+	// case OREGISTER:
-		if p.bool() {
+	//	unimplemented
-			n.Sym = p.sym()
-		} else {
-			n.Type = p.typ()
-		}
 
 	case OLITERAL:
 		typ := p.typ()
-		n.Type = idealType(typ)
+		n := nodlit(p.value(typ))
-		n.SetVal(p.value(typ))
+		if !typ.IsUntyped() {
-
+			conv := Nod(OCALL, typenod(typ), nil)
-	// expressions
+			conv.List.Set1(n)
-	case OMAKEMAP, OMAKECHAN, OMAKESLICE:
+			n = conv
-		if p.bool() {
-			n.List.Set(p.nodeList())
 		}
-		n.Left, n.Right = p.nodesOrNil()
+		return n
-		n.Type = p.typ()
 
-	case OPLUS, OMINUS, OADDR, OCOM, OIND, ONOT, ORECV:
+	case ONAME:
-		n.Left = p.node()
+		if p.bool() {
+			// "_"
+			// TODO(gri) avoid repeated "_" lookup
+			return mkname(Pkglookup("_", localpkg))
+		}
+		return NodSym(OXDOT, typenod(p.typ()), p.fieldSym())
 
-	case OADD, OAND, OANDAND, OANDNOT, ODIV, OEQ, OGE, OGT, OLE, OLT,
+	case OPACK, ONONAME:
-		OLSH, OMOD, OMUL, ONE, OOR, OOROR, ORSH, OSEND,
+		return mkname(p.sym())
-		OSUB, OXOR:
-		n.Left = p.node()
-		n.Right = p.node()
 
-	case OADDSTR:
+	case OTYPE:
-		n.List.Set(p.nodeList())
+		if p.bool() {
+			return mkname(p.sym())
+		}
+		return typenod(p.typ())
+
+	// case OTARRAY, OTMAP, OTCHAN, OTSTRUCT, OTINTER, OTFUNC:
+	//      unreachable - should have been resolved by typechecking
+
+	// case OCLOSURE:
+	//	unimplemented
+
+	// case OCOMPLIT:
+	//	unimplemented
 
 	case OPTRLIT:
-		n.Left = p.node()
+		n := p.expr()
+		if !p.bool() /* !implicit, i.e. '&' operator*/ {
+			if n.Op == OCOMPLIT {
+				// Special case for &T{...}: turn into (*T){...}.
+				n.Right = Nod(OIND, n.Right, nil)
+				n.Right.Implicit = true
+			} else {
+				n = Nod(OADDR, n, nil)
+			}
+		}
+		return n
 
 	case OSTRUCTLIT:
-		n.Type = p.typ()
+		n := Nod(OCOMPLIT, nil, nil)
-		n.List.Set(p.nodeList())
+		if !p.bool() {
-		n.Implicit = p.bool()
+			n.Right = typenod(p.typ())
+		}
+		n.List.Set(p.elemList())
+		return n
 
 	case OARRAYLIT, OMAPLIT:
-		n.Type = p.typ()
+		n := Nod(OCOMPLIT, nil, nil)
-		n.List.Set(p.nodeList())
+		if !p.bool() {
-		n.Implicit = p.bool()
+			n.Right = typenod(p.typ())
+		}
+		n.List.Set(p.exprList())
+		return n
 
 	case OKEY:
-		n.Left, n.Right = p.nodesOrNil()
+		left, right := p.exprsOrNil()
+		return Nod(OKEY, left, right)
 
-	case OCOPY, OCOMPLEX:
+	// case OCALLPART:
-		n.Left = p.node()
+	//	unimplemented
-		n.Right = p.node()
 
-	case OCONV, OCONVIFACE, OCONVNOP, OARRAYBYTESTR, OARRAYRUNESTR, OSTRARRAYBYTE, OSTRARRAYRUNE, ORUNESTR:
+	// case OXDOT, ODOT, ODOTPTR, ODOTINTER, ODOTMETH:
-		// n.Type = p.typ()
+	// 	unreachable - mapped to case OXDOT below by exporter
-		// if p.bool() {
-		// 	n.Left = p.node()
-		// } else {
-		// 	n.List.Set(p.nodeList())
-		// }
-		x := Nod(OCALL, p.typ().Nod, nil)
-		if p.bool() {
-			x.List.Set1(p.node())
-		} else {
-			x.List.Set(p.nodeList())
-		}
-		return x
 
-	case ODOT, ODOTPTR, ODOTMETH, ODOTINTER, OXDOT:
+	case OXDOT:
 		// see parser.new_dotname
-		obj := p.node()
+		obj := p.expr()
-		sel := p.sym()
+		sel := p.fieldSym()
 		if obj.Op == OPACK {
 			s := restrictlookup(sel.Name, obj.Name.Pkg)
 			obj.Used = true
@@ -659,123 +783,253 @@ func (p *importer) node() *Node {
 		}
 		return NodSym(OXDOT, obj, sel)
 
-	case ODOTTYPE, ODOTTYPE2:
+	// case ODOTTYPE, ODOTTYPE2:
-		n.Left = p.node()
+	// 	unreachable - mapped to case ODOTTYPE below by exporter
+
+	case ODOTTYPE:
+		n := Nod(ODOTTYPE, p.expr(), nil)
 		if p.bool() {
-			n.Right = p.node()
+			n.Right = p.expr()
 		} else {
-			n.Type = p.typ()
+			n.Right = typenod(p.typ())
 		}
+		return n
 
-	case OINDEX, OINDEXMAP, OSLICE, OSLICESTR, OSLICEARR, OSLICE3, OSLICE3ARR:
+	// case OINDEX, OINDEXMAP, OSLICE, OSLICESTR, OSLICEARR, OSLICE3, OSLICE3ARR:
-		n.Left = p.node()
+	// 	unreachable - mapped to cases below by exporter
-		n.Right = p.node()
 
-	case OREAL, OIMAG, OAPPEND, OCAP, OCLOSE, ODELETE, OLEN, OMAKE, ONEW, OPANIC,
+	case OINDEX, OSLICE, OSLICE3:
-		ORECOVER, OPRINT, OPRINTN:
+		return Nod(op, p.expr(), p.expr())
-		n.Left, _ = p.nodesOrNil()
+
-		n.List.Set(p.nodeList())
+	case OCOPY, OCOMPLEX:
+		n := builtinCall(op)
+		n.List.Set([]*Node{p.expr(), p.expr()})
+		return n
+
+	// case OCONV, OCONVIFACE, OCONVNOP, OARRAYBYTESTR, OARRAYRUNESTR, OSTRARRAYBYTE, OSTRARRAYRUNE, ORUNESTR:
+	// 	unreachable - mapped to OCONV case below by exporter
+
+	case OCONV:
+		n := Nod(OCALL, typenod(p.typ()), nil)
+		if p.bool() {
+			n.List.Set1(p.expr())
+		} else {
+			n.List.Set(p.exprList())
+		}
+		return n
+
+	case OREAL, OIMAG, OAPPEND, OCAP, OCLOSE, ODELETE, OLEN, OMAKE, ONEW, OPANIC, ORECOVER, OPRINT, OPRINTN:
+		n := builtinCall(op)
+		if p.bool() {
+			n.List.Set1(p.expr())
+		} else {
+			n.List.Set(p.exprList())
+			n.Isddd = p.bool()
+		}
+		return n
+
+	// case OCALL, OCALLFUNC, OCALLMETH, OCALLINTER, OGETG:
+	// 	unreachable - mapped to OCALL case below by exporter
+
+	case OCALL:
+		n := Nod(OCALL, p.expr(), nil)
+		n.List.Set(p.exprList())
 		n.Isddd = p.bool()
+		return n
 
-	case OCALL, OCALLFUNC, OCALLMETH, OCALLINTER, OGETG:
+	case OMAKEMAP, OMAKECHAN, OMAKESLICE:
-		n.Left = p.node()
+		n := builtinCall(OMAKE)
-		n.List.Set(p.nodeList())
+		n.List.Append(typenod(p.typ()))
-		n.Isddd = p.bool()
+		n.List.Append(p.exprList()...)
+		return n
 
-	case OCMPSTR, OCMPIFACE:
+	// unary expressions
-		n.Left = p.node()
+	case OPLUS, OMINUS, OADDR, OCOM, OIND, ONOT, ORECV:
-		n.Right = p.node()
+		return Nod(op, p.expr(), nil)
-		n.Etype = EType(p.int())
 
-	case OPAREN:
+	// binary expressions
-		n.Left = p.node()
+	case OADD, OAND, OANDAND, OANDNOT, ODIV, OEQ, OGE, OGT, OLE, OLT,
+		OLSH, OMOD, OMUL, ONE, OOR, OOROR, ORSH, OSEND, OSUB, OXOR:
+		return Nod(op, p.expr(), p.expr())
 
+	case OADDSTR:
+		list := p.exprList()
+		x := list[0]
+		for _, y := range list[1:] {
+			x = Nod(OADD, x, y)
+		}
+		return x
+
+	// case OCMPSTR, OCMPIFACE:
+	// 	unreachable - mapped to std comparison operators by exporter
+
+	case ODCLCONST:
+		// TODO(gri) these should not be exported in the first place
+		return Nod(OEMPTY, nil, nil)
+
+	// --------------------------------------------------------------------
 	// statements
 	case ODCL:
-		n.Left = p.node() // TODO(gri) compare with fmt code
+		var lhs *Node
-		n.Left.Type = p.typ()
+		if p.bool() {
+			lhs = p.expr()
+		} else {
+			lhs = dclname(p.sym())
+		}
+		// TODO(gri) avoid list created here!
+		return liststmt(variter([]*Node{lhs}, typenod(p.typ()), nil))
 
-	case OAS:
+	// case ODCLFIELD:
-		n.Left, n.Right = p.nodesOrNil()
+	//	unimplemented
-		n.Colas = p.bool() // TODO(gri) what about complexinit?
+
+	case OAS, OASWB:
+		if p.bool() {
+			lhs := p.expr()
+			rhs := p.expr()
+			return Nod(OAS, lhs, rhs)
+		}
+		// TODO(gri) we should not have emitted anything here
+		return Nod(OEMPTY, nil, nil)
 
 	case OASOP:
-		n.Left = p.node()
+		n := Nod(OASOP, nil, nil)
-		n.Right = p.node()
 		n.Etype = EType(p.int())
+		n.Left = p.expr()
+		if !p.bool() {
+			n.Right = Nodintconst(1)
+			n.Implicit = true
+		} else {
+			n.Right = p.expr()
+		}
+		return n
 
-	case OAS2, OASWB:
+	case OAS2:
-		n.List.Set(p.nodeList())
+		lhs := p.exprList()
-		n.Rlist.Set(p.nodeList())
+		rhs := p.exprList()
+		n := Nod(OAS2, nil, nil)
+		n.List.Set(lhs)
+		n.Rlist.Set(rhs)
+		return n
 
 	case OAS2DOTTYPE, OAS2FUNC, OAS2MAPR, OAS2RECV:
-		n.List.Set(p.nodeList())
+		n := Nod(OAS2, nil, nil)
-		n.Rlist.Set(p.nodeList())
+		n.List.Set(p.exprList())
+		n.Rlist.Set(p.exprList())
+		return n
 
 	case ORETURN:
-		n.List.Set(p.nodeList())
+		n := Nod(ORETURN, nil, nil)
+		n.List.Set(p.exprList())
+		return n
+
+	// case ORETJMP:
+	// 	unreachable - generated by compiler for trampolin routines (not exported)
 
 	case OPROC, ODEFER:
-		n.Left = p.node()
+		return Nod(op, p.expr(), nil)
 
 	case OIF:
-		n.Ninit.Set(p.nodeList())
+		markdcl()
-		n.Left = p.node()
+		n := Nod(OIF, nil, nil)
-		n.Nbody.Set(p.nodeList())
+		n.Ninit.Set(p.stmtList())
-		n.Rlist.Set(p.nodeList())
+		n.Left = p.expr()
+		n.Nbody.Set(p.stmtList())
+		n.Rlist.Set(p.stmtList())
+		popdcl()
+		return n
 
 	case OFOR:
-		n.Ninit.Set(p.nodeList())
+		markdcl()
-		n.Left, n.Right = p.nodesOrNil()
+		n := Nod(OFOR, nil, nil)
-		n.Nbody.Set(p.nodeList())
+		n.Ninit.Set(p.stmtList())
+		n.Left, n.Right = p.exprsOrNil()
+		n.Nbody.Set(p.stmtList())
+		popdcl()
+		return n
 
 	case ORANGE:
-		if p.bool() {
+		markdcl()
-			n.List.Set(p.nodeList())
+		n := Nod(ORANGE, nil, nil)
-		}
+		n.List.Set(p.stmtList())
-		n.Right = p.node()
+		n.Right = p.expr()
-		n.Nbody.Set(p.nodeList())
+		n.Nbody.Set(p.stmtList())
+		popdcl()
+		return n
 
 	case OSELECT, OSWITCH:
-		n.Ninit.Set(p.nodeList())
+		markdcl()
-		n.Left, _ = p.nodesOrNil()
+		n := Nod(op, nil, nil)
-		n.List.Set(p.nodeList())
+		n.Ninit.Set(p.stmtList())
+		n.Left, _ = p.exprsOrNil()
+		n.List.Set(p.stmtList())
+		popdcl()
+		return n
 
 	case OCASE, OXCASE:
-		if p.bool() {
+		markdcl()
-			n.List.Set(p.nodeList())
+		n := Nod(OXCASE, nil, nil)
-		}
+		n.List.Set(p.exprList())
-		n.Nbody.Set(p.nodeList())
+		// TODO(gri) eventually we must declare variables for type switch
+		// statements (type switch statements are not yet exported)
+		n.Nbody.Set(p.stmtList())
+		popdcl()
+		return n
 
 	case OBREAK, OCONTINUE, OGOTO, OFALL, OXFALL:
-		n.Left, _ = p.nodesOrNil()
+		if op == OFALL {
+			op = OXFALL
+		}
+		left, _ := p.exprsOrNil()
+		return Nod(op, left, nil)
 
-	case OEMPTY, ODCLCONST:
+	// case OEMPTY:
-		// nothing to do
+	// 	unreachable - not emitted by exporter
 
 	case OLABEL:
-		n.Left = p.node()
+		n := Nod(OLABEL, p.expr(), nil)
+		n.Left.Sym = dclstack // context, for goto restrictions
+		return n
+
+	case OEND:
+		return nil
 
 	default:
-		panic(fmt.Sprintf("importer: %s (%d) node not yet supported", opnames[n.Op], n.Op))
+		Fatalf("importer: %s (%d) node not yet supported", opnames[op], op)
+		panic("unreachable") // satisfy compiler
 	}
-
-	return n
 }
 
-func (p *importer) nodesOrNil() (a, b *Node) {
+func builtinCall(op Op) *Node {
+	return Nod(OCALL, mkname(builtinpkg.Lookup(goopnames[op])), nil)
+}
+
+func (p *importer) exprsOrNil() (a, b *Node) {
 	ab := p.int()
 	if ab&1 != 0 {
-		a = p.node()
+		a = p.expr()
 	}
 	if ab&2 != 0 {
-		b = p.node()
+		b = p.expr()
 	}
 	return
 }
 
+func (p *importer) fieldSym() *Sym {
+	name := p.string()
+	pkg := localpkg
+	if !exportname(name) {
+		pkg = p.pkg()
+	}
+	return pkg.Lookup(name)
+}
+
 func (p *importer) sym() *Sym {
-	return p.fieldName()
+	name := p.string()
+	pkg := localpkg
+	if name != "_" {
+		pkg = p.pkg()
+	}
+	return pkg.Lookup(name)
 }
 
 func (p *importer) bool() bool {
@@ -818,6 +1072,8 @@ func (p *importer) string() string {
 		p.marker('s')
 	}
 
+	// TODO(gri) should we intern strings here?
+
 	if n := int(p.rawInt64()); n > 0 {
 		if cap(p.buf) < n {
 			p.buf = make([]byte, n)

src/cmd/compile/internal/gc/dcl.go

@@ -79,18 +79,17 @@ func markdcl() {
 	block = blockgen
 }
 
-func dumpdcl(st string) {
+// keep around for debugging
+func dumpdclstack() {
 	i := 0
 	for d := dclstack; d != nil; d = d.Link {
-		i++
+		fmt.Printf("%6d  %p", i, d)
-		fmt.Printf("    %.2d %p", i, d)
+		if d.Name != "" {
-		if d.Name == "" {
+			fmt.Printf("  '%s'  %v\n", d.Name, Pkglookup(d.Name, d.Pkg))
-			fmt.Printf("\n")
+		} else {
-			continue
+			fmt.Printf("  ---\n")
 		}
-
+		i++
-		fmt.Printf(" '%s'", d.Name)
-		fmt.Printf(" %v\n", Pkglookup(d.Name, d.Pkg))
 	}
 }
 
@@ -250,11 +249,8 @@ func variter(vl []*Node, t *Node, el []*Node) []*Node {
 				Yyerror("missing expression in var declaration")
 				break
 			}
-
 			e = el[0]
 			el = el[1:]
-		} else {
-			e = nil
 		}
 
 		v.Op = ONAME
@@ -527,7 +523,7 @@ func ifacedcl(n *Node) {
 func funchdr(n *Node) {
 	// change the declaration context from extern to auto
 	if Funcdepth == 0 && dclcontext != PEXTERN {
-		Fatalf("funchdr: dclcontext")
+		Fatalf("funchdr: dclcontext = %d", dclcontext)
 	}
 
 	if importpkg == nil && n.Func.Nname != nil {
@@ -678,7 +674,7 @@ func funcargs2(t *Type) {
 func funcbody(n *Node) {
 	// change the declaration context from auto to extern
 	if dclcontext != PAUTO {
-		Fatalf("funcbody: dclcontext")
+		Fatalf("funcbody: unexpected dclcontext %d", dclcontext)
 	}
 	popdcl()
 	Funcdepth--

src/cmd/compile/internal/gc/export.go

@@ -380,9 +380,8 @@ func dumpexport() {
 	if forceNewExport || newexport != 0 {
 		// binary export
 		// The linker also looks for the $$ marker - use char after $$ to distinguish format.
-		exportf("\n$$B\n")        // indicate binary format
+		exportf("\n$$B\n") // indicate binary format
-		const verifyExport = true // enable to check format changes
+		if debugFormat {
-		if verifyExport {
 			// save a copy of the export data
 			var copy bytes.Buffer
 			bcopy := obj.Binitw(&copy)
@@ -430,9 +429,8 @@ func dumpexport() {
 		}
 
 		// exportlist grows during iteration - cannot use range
-		for len(exportlist) > 0 {
+		for i := 0; i < len(exportlist); i++ {
-			n := exportlist[0]
+			n := exportlist[i]
-			exportlist = exportlist[1:]
 			lineno = n.Lineno
 			dumpsym(n.Sym)
 		}

src/cmd/compile/internal/gc/fmt.go

@@ -801,7 +801,7 @@ func stmtfmt(n *Node) string {
 		}
 
 	// Don't export "v = <N>" initializing statements, hope they're always
-	// preceded by the DCL which will be re-parsed and typecheck to reproduce
+	// preceded by the DCL which will be re-parsed and typechecked to reproduce
 	// the "v = <N>" again.
 	case OAS, OASWB:
 		if fmtmode == FExp && n.Right == nil {
@@ -1146,9 +1146,7 @@ func exprfmt(n *Node, prec int) string {
 		if n.Left != nil {
 			return fmt.Sprintf("[]%v", n.Left)
 		}
-		var f string
+		return fmt.Sprintf("[]%v", n.Right) // happens before typecheck
-		f += fmt.Sprintf("[]%v", n.Right)
-		return f // happens before typecheck
 
 	case OTMAP:
 		return fmt.Sprintf("map[%v]%v", n.Left, n.Right)

src/cmd/compile/internal/gc/inl.go

@@ -66,7 +66,7 @@ func typecheckinl(fn *Node) {
 		return // typecheckinl on local function
 	}
 
-	if Debug['m'] > 2 {
+	if Debug['m'] > 2 || Debug_export != 0 {
 		fmt.Printf("typecheck import [%v] %v { %v }\n", fn.Sym, Nconv(fn, FmtLong), Hconv(fn.Func.Inl, FmtSharp))
 	}
 

src/cmd/compile/internal/gc/main.go

@@ -815,6 +815,9 @@ func importfile(f *Val, indent []byte) {
 
 	case 'B':
 		// new export format
+		if Debug_export != 0 {
+			fmt.Printf("importing %s (%s)\n", path_, file)
+		}
 		imp.ReadByte() // skip \n after $$B
 		Import(imp)
 

src/cmd/compile/internal/gc/parser.go

@@ -1588,7 +1588,7 @@ func (p *parser) onew_name() *Node {
 func (p *parser) sym() *Sym {
 	switch p.tok {
 	case LNAME:
-		s := p.sym_
+		s := p.sym_ // from localpkg
 		p.next()
 		// during imports, unqualified non-exported identifiers are from builtinpkg
 		if importpkg != nil && !exportname(s.Name) {

src/cmd/compile/internal/gc/pgen.go

@@ -490,6 +490,12 @@ func compile(fn *Node) {
 	}
 }
 
+type symByName []*Sym
+
+func (a symByName) Len() int           { return len(a) }
+func (a symByName) Less(i, j int) bool { return a[i].Name < a[j].Name }
+func (a symByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
 // genlegacy compiles Curfn using the legacy non-SSA code generator.
 func genlegacy(ptxt *obj.Prog, gcargs, gclocals *Sym) {
 	Genlist(Curfn.Func.Enter)

src/go/internal/gcimporter/bimport.go

@@ -15,6 +15,18 @@ import (
 	"unicode/utf8"
 )
 
+type importer struct {
+	imports  map[string]*types.Package
+	data     []byte
+	buf      []byte   // for reading strings
+	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
+	pkgList  []*types.Package
+	typList  []types.Type
+
+	debugFormat bool
+	read        int // bytes read
+}
+
 // BImportData imports a package from the serialized package data
 // and returns the number of bytes consumed and a reference to the package.
 // If data is obviously malformed, an error is returned but in
@@ -39,7 +51,7 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 	// --- generic export data ---
 
 	if v := p.string(); v != "v0" {
-		return p.read, nil, fmt.Errorf("unknown version: %s", v)
+		return p.read, nil, fmt.Errorf("unknown export data version: %s", v)
 	}
 
 	// populate typList with predeclared "known" types
@@ -69,37 +81,20 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 	// read compiler-specific flags
 	p.string() // discard
 
-	// read consts
+	// read objects of phase 1 only (see cmd/compiler/internal/gc/bexport.go)
-	for i := p.int(); i > 0; i-- {
+	objcount := 0
-		name := p.string()
+	for {
-		typ := p.typ(nil)
+		tag := p.tagOrIndex()
-		val := p.value()
+		if tag == endTag {
-		p.declare(types.NewConst(token.NoPos, pkg, name, typ, val))
+			break
+		}
+		p.obj(tag)
+		objcount++
 	}
 
-	// read vars
+	// self-verification
-	for i := p.int(); i > 0; i-- {
+	if count := p.int(); count != objcount {
-		name := p.string()
+		panic(fmt.Sprintf("importer: got %d objects; want %d", objcount, count))
-		typ := p.typ(nil)
-		p.declare(types.NewVar(token.NoPos, pkg, name, typ))
-	}
-
-	// read funcs
-	for i := p.int(); i > 0; i-- {
-		name := p.string()
-		params, isddd := p.paramList()
-		result, _ := p.paramList()
-		sig := types.NewSignature(nil, params, result, isddd)
-		p.int() // read and discard index of inlined function body
-		p.declare(types.NewFunc(token.NoPos, pkg, name, sig))
-	}
-
-	// read types
-	for i := p.int(); i > 0; i-- {
-		// name is parsed as part of named type and the
-		// type object is added to scope via respective
-		// named type
-		_ = p.typ(nil).(*types.Named)
 	}
 
 	// ignore compiler-specific import data
@@ -122,25 +117,6 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 	return p.read, pkg, nil
 }
 
-type importer struct {
-	imports  map[string]*types.Package
-	data     []byte
-	buf      []byte   // for reading strings
-	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
-	pkgList  []*types.Package
-	typList  []types.Type
-
-	debugFormat bool
-	read        int // bytes read
-}
-
-func (p *importer) declare(obj types.Object) {
-	if alt := p.pkgList[0].Scope().Insert(obj); alt != nil {
-		// This can only happen if we import a package a second time.
-		panic(fmt.Sprintf("%s already declared", alt.Name()))
-	}
-}
-
 func (p *importer) pkg() *types.Package {
 	// if the package was seen before, i is its index (>= 0)
 	i := p.tagOrIndex()
@@ -178,6 +154,55 @@ func (p *importer) pkg() *types.Package {
 	return pkg
 }
 
+func (p *importer) declare(obj types.Object) {
+	pkg := obj.Pkg()
+	if alt := pkg.Scope().Insert(obj); alt != nil {
+		// This could only trigger if we import a (non-type) object a second time.
+		// This should never happen because 1) we only import a package once; and
+		// b) we ignore compiler-specific export data which may contain functions
+		// whose inlined function bodies refer to other functions that were already
+		// imported.
+		// (See also the comment in cmd/compile/internal/gc/bimport.go importer.obj,
+		// switch case importing functions).
+		panic(fmt.Sprintf("%s already declared", alt.Name()))
+	}
+}
+
+func (p *importer) obj(tag int) {
+	switch tag {
+	case constTag:
+		pkg, name := p.qualifiedName()
+		typ := p.typ(nil)
+		val := p.value()
+		p.declare(types.NewConst(token.NoPos, pkg, name, typ, val))
+
+	case typeTag:
+		_ = p.typ(nil)
+
+	case varTag:
+		pkg, name := p.qualifiedName()
+		typ := p.typ(nil)
+		p.declare(types.NewVar(token.NoPos, pkg, name, typ))
+
+	case funcTag:
+		pkg, name := p.qualifiedName()
+		params, isddd := p.paramList()
+		result, _ := p.paramList()
+		sig := types.NewSignature(nil, params, result, isddd)
+		p.int() // read and discard index of inlined function body
+		p.declare(types.NewFunc(token.NoPos, pkg, name, sig))
+
+	default:
+		panic("unexpected object tag")
+	}
+}
+
+func (p *importer) qualifiedName() (pkg *types.Package, name string) {
+	name = p.string()
+	pkg = p.pkg()
+	return
+}
+
 func (p *importer) record(t types.Type) {
 	p.typList = append(p.typList, t)
 }
@@ -239,11 +264,17 @@ func (p *importer) typ(parent *types.Package) types.Type {
 
 		// read associated methods
 		for i := p.int(); i > 0; i-- {
+			// TODO(gri) replace this with something closer to fieldName
 			name := p.string()
+			if !exported(name) {
+				p.pkg()
+			}
+
 			recv, _ := p.paramList() // TODO(gri) do we need a full param list for the receiver?
 			params, isddd := p.paramList()
 			result, _ := p.paramList()
 			p.int() // read and discard index of inlined function body
+
 			sig := types.NewSignature(recv.At(0), params, result, isddd)
 			t0.AddMethod(types.NewFunc(token.NoPos, parent, name, sig))
 		}
@@ -432,18 +463,20 @@ func (p *importer) param(named bool) (*types.Var, bool) {
 		t = types.NewSlice(td.elem)
 	}
 
+	var pkg *types.Package
 	var name string
 	if named {
 		name = p.string()
 		if name == "" {
 			panic("expected named parameter")
 		}
+		pkg = p.pkg()
 	}
 
 	// read and discard compiler-specific info
 	p.string()
 
-	return types.NewVar(token.NoPos, nil, name, t), isddd
+	return types.NewVar(token.NoPos, pkg, name, t), isddd
 }
 
 func exported(name string) bool {
@@ -617,8 +650,13 @@ func (p *importer) byte() byte {
 
 // Tags. Must be < 0.
 const (
-	// Packages
+	// Objects
 	packageTag = -(iota + 1)
+	constTag
+	typeTag
+	varTag
+	funcTag
+	endTag
 
 	// Types
 	namedTag
@@ -640,6 +678,7 @@ const (
 	fractionTag // not used by gc
 	complexTag
 	stringTag
+	unknownTag // not used by gc (only appears in packages with errors)
 )
 
 var predeclared = []types.Type{