[dev.link] cmd/link: convert DWARF type generation to use loader

Converts the portion of DWARF generation that deals with creation of
type DIEs and constant DIEs to use the new loader interfaces. Creation
of subprogram DIE and compilation unit DIE content still operates on
sym.Symbols at the moment, and happens much later in the linker.

The new code for type DIE generation is gated/guarded by the linker
flag "-newdw", which currently defaults to true. At some point in the
near future this flag should be removed, but it is handy for triage at
the moment.

This patch also includes shim code designed to run after loadlibfull()
that walks through the DIE chains and to converts loader.Sym
references back into sym.Symbol references for the remainder of the
compilation, since the second phase of DWARF has not yet been
converted.

Change-Id: I681a00fb8a1f3c37884a79b373d86411332e07c8
Reviewed-on: https://go-review.googlesource.com/c/go/+/208230
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Jeremy Faller <jeremy@golang.org>

src/cmd/link/internal/ld/deadcode.go

@@ -68,7 +68,10 @@ func addToTextp(ctxt *Link) {
 
 	// Put reachable text symbols into Textp.
 	// do it in postorder so that packages are laid down in dependency order
-	// internal first, then everything else
+	// internal first, then everything else. This also populates lib and
+	// unit Textp slices, which are needed for DWARF
+	// FIXME: once DWARF is completely converted to using loader.Sym,
+	// we can remove the *sym.Symbol Textp slices.
 	ctxt.Library = postorder(ctxt.Library)
 	for _, doInternal := range [2]bool{true, false} {
 		for _, lib := range ctxt.Library {
@@ -76,23 +79,21 @@ func addToTextp(ctxt *Link) {
 				continue
 			}
 			libtextp := lib.Textp[:0]
-			for idx, s := range lib.Textp {
+			for _, s := range lib.Textp {
 				if s.Attr.Reachable() {
 					textp = append(textp, s)
 					libtextp = append(libtextp, s)
 					if s.Unit != nil {
 						s.Unit.Textp = append(s.Unit.Textp, s)
-						s.Unit.Textp2 = append(s.Unit.Textp2, lib.Textp2[idx])
 					}
 				}
 			}
-			for idx, s := range lib.DupTextSyms {
+			for _, s := range lib.DupTextSyms {
 				if s.Attr.Reachable() && !s.Attr.OnList() {
 					textp = append(textp, s)
 					libtextp = append(libtextp, s)
 					if s.Unit != nil {
 						s.Unit.Textp = append(s.Unit.Textp, s)
-						s.Unit.Textp2 = append(s.Unit.Textp2, lib.DupTextSyms2[idx])
 					}
 					s.Attr |= sym.AttrOnList
 					// dupok symbols may be defined in multiple packages. its

src/cmd/link/internal/ld/lib.go

@@ -2620,6 +2620,13 @@ func (ctxt *Link) loadlibfull() {
 	// Pull the symbols out.
 	ctxt.loader.ExtractSymbols(ctxt.Syms, ctxt.Reachparent)
 
+	// If -newdw is in effect, then we generated dwarf DIE objects
+	// with embedded loader.Sym refs as opposed to sym.Symbol refs.
+	// Call a helper to rewrite the former to the latter in all DIEs
+	if *FlagNewDw {
+		dwarfConvertSymbols(ctxt)
+	}
+
 	setupdynexp(ctxt)
 
 	// Drop the cgodata reference.

src/cmd/link/internal/ld/main.go

@@ -88,6 +88,7 @@ var (
 	flagInterpreter = flag.String("I", "", "use `linker` as ELF dynamic linker")
 	FlagDebugTramp  = flag.Int("debugtramp", 0, "debug trampolines")
 	FlagStrictDups  = flag.Int("strictdups", 0, "sanity check duplicate symbol contents during object file reading (1=warn 2=err).")
+	FlagNewDw       = flag.Bool("newdw", true, "DWARF gen with new loader")
 
 	FlagRound       = flag.Int("R", -1, "set address rounding `quantum`")
 	FlagTextAddr    = flag.Int64("T", -1, "set text segment `address`")
@@ -241,11 +242,31 @@ func Main(arch *sys.Arch, theArch Arch) {
 		fieldtrack(ctxt.Arch, ctxt.loader)
 	}
 
+	if *FlagNewDw {
+		bench.Start("dwarfGenerateDebugInfo")
+
+		// DWARF-gen requires that the unit Textp2 slices be populated,
+		// so that it can walk the functions in each unit. Call into
+		// the loader to do this (requires that we collect the set of
+		// internal libraries first). NB: might be simpler if we moved
+		// isRuntimeDepPkg to cmd/internal and then did the test
+		// in loader.AssignTextSymbolOrder.
+		ctxt.Library = postorder(ctxt.Library)
+		intlibs := []bool{}
+		for _, lib := range ctxt.Library {
+			intlibs = append(intlibs, isRuntimeDepPkg(lib.Pkg))
+		}
+		ctxt.loader.AssignTextSymbolOrder(ctxt.Library, intlibs)
+		dwarfGenerateDebugInfo(ctxt)
+	}
+
 	bench.Start("loadlibfull")
 	ctxt.loadlibfull() // XXX do it here for now
 
-	bench.Start("dwarfGenerateDebugInfo")
-	dwarfGenerateDebugInfo(ctxt)
+	if !*FlagNewDw {
+		bench.Start("dwarfGenerateDebugInfo")
+		dwarfGenerateDebugInfo(ctxt)
+	}
 
 	bench.Start("mangleTypeSym")
 	ctxt.mangleTypeSym()

src/cmd/link/internal/loader/loader.go

@@ -227,7 +227,7 @@ type Loader struct {
 }
 
 const (
-	pkgDef    = iota
+	pkgDef = iota
 	nonPkgDef
 	nonPkgRef
 )
@@ -1239,6 +1239,56 @@ func (l *Loader) AuxSym(i Sym, j int) Sym {
 	return l.resolve(r, a.Sym)
 }
 
+// GetFuncDwarfAuxSyms collects and returns the auxiliary DWARF
+// symbols associated with a given function symbol.  Prior to the
+// introduction of the loader, this was done purely using name
+// lookups, e.f. for function with name XYZ we would then look up
+// go.info.XYZ, etc.
+// FIXME: once all of dwarfgen is converted over to the loader,
+// it would save some space to make these aux symbols nameless.
+func (l *Loader) GetFuncDwarfAuxSyms(fnSymIdx Sym) (auxDwarfInfo, auxDwarfLoc, auxDwarfRanges, auxDwarfLines Sym) {
+	if l.SymType(fnSymIdx) != sym.STEXT {
+		log.Fatalf("error: non-function sym %d/%s t=%s passed to GetFuncDwarfAuxSyms", fnSymIdx, l.SymName(fnSymIdx), l.SymType(fnSymIdx).String())
+	}
+	if l.IsExternal(fnSymIdx) {
+		// Current expectation is that any external function will
+		// not have auxsyms.
+		return
+	}
+	naux := l.NAux(fnSymIdx)
+	if naux == 0 {
+		return
+	}
+	r, li := l.toLocal(fnSymIdx)
+	for i := 0; i < naux; i++ {
+		a := goobj2.Aux{}
+		a.Read(r.Reader, r.AuxOff(li, i))
+		switch a.Type {
+		case goobj2.AuxDwarfInfo:
+			auxDwarfInfo = l.resolve(r, a.Sym)
+			if l.SymType(auxDwarfInfo) != sym.SDWARFINFO {
+				panic("aux dwarf info sym with wrong type")
+			}
+		case goobj2.AuxDwarfLoc:
+			auxDwarfLoc = l.resolve(r, a.Sym)
+			if l.SymType(auxDwarfLoc) != sym.SDWARFLOC {
+				panic("aux dwarf loc sym with wrong type")
+			}
+		case goobj2.AuxDwarfRanges:
+			auxDwarfRanges = l.resolve(r, a.Sym)
+			if l.SymType(auxDwarfRanges) != sym.SDWARFRANGE {
+				panic("aux dwarf ranges sym with wrong type")
+			}
+		case goobj2.AuxDwarfLines:
+			auxDwarfLines = l.resolve(r, a.Sym)
+			if l.SymType(auxDwarfLines) != sym.SDWARFLINES {
+				panic("aux dwarf lines sym with wrong type")
+			}
+		}
+	}
+	return
+}
+
 // ReadAuxSyms reads the aux symbol ids for the specified symbol into the
 // slice passed as a parameter. If the slice capacity is not large enough, a new
 // larger slice will be allocated. Final slice is returned.
@@ -1545,6 +1595,7 @@ func (l *Loader) preloadSyms(r *oReader, kind int) {
 			}
 		}
 		if strings.HasPrefix(name, "go.string.") ||
+			strings.HasPrefix(name, "gclocals·") ||
 			strings.HasPrefix(name, "runtime.gcbits.") {
 			l.SetAttrNotInSymbolTable(gi, true)
 		}
@@ -2019,7 +2070,6 @@ func loadObjFull(l *Loader, r *oReader) {
 				s := l.Syms[gi]
 				if s.Type == sym.STEXT {
 					lib.DupTextSyms = append(lib.DupTextSyms, s)
-					lib.DupTextSyms2 = append(lib.DupTextSyms2, sym.LoaderSym(gi))
 				}
 			}
 			continue
@@ -2211,12 +2261,10 @@ func loadObjFull(l *Loader, r *oReader) {
 			}
 			s.Attr.Set(sym.AttrOnList, true)
 			lib.Textp = append(lib.Textp, s)
-			lib.Textp2 = append(lib.Textp2, sym.LoaderSym(isym))
 		} else {
 			// there may be a dup in another package
 			// put into a temp list and add to text later
 			lib.DupTextSyms = append(lib.DupTextSyms, s)
-			lib.DupTextSyms2 = append(lib.DupTextSyms2, sym.LoaderSym(isym))
 		}
 	}
 }
@@ -2322,6 +2370,83 @@ func (l *Loader) UndefinedRelocTargets(limit int) []Sym {
 	return result
 }
 
+// AssignTextSymbolOrder populates the Textp2 slices within each
+// library and compilation unit, insuring that packages are laid down
+// in dependency order (internal first, then everything else).
+func (l *Loader) AssignTextSymbolOrder(libs []*sym.Library, intlibs []bool) {
+
+	// Library Textp2 lists should be empty at this point.
+	for _, lib := range libs {
+		if len(lib.Textp2) != 0 {
+			panic("expected empty Textp2 slice for library")
+		}
+		if len(lib.DupTextSyms2) != 0 {
+			panic("expected empty DupTextSyms2 slice for library")
+		}
+	}
+
+	// Used to record which dupok symbol we've assigned to a unit.
+	// Can't use the onlist attribute here because it will need to
+	// clear for the later assignment of the sym.Symbol to a unit.
+	// NB: we can convert to using onList once we no longer have to
+	// call the regular addToTextp.
+	assignedToUnit := makeBitmap(l.NSym() + 1)
+
+	// Walk through all text symbols from Go object files and append
+	// them to their corresponding library's textp2 list.
+	for _, o := range l.objs[1:] {
+		r := o.r
+		lib := r.unit.Lib
+		for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ {
+			gi := l.toGlobal(r, i)
+			osym := goobj2.Sym{}
+			osym.ReadWithoutName(r.Reader, r.SymOff(i))
+			st := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]
+			if st != sym.STEXT {
+				continue
+			}
+			// check for dupok
+			if r2, i2 := l.toLocal(gi); r2 != r || i2 != i {
+				if l.attrReachable.has(gi) {
+					// A dupok symbol is resolved to another package.
+					// We still need to record its presence in the
+					// current package, as the trampoline pass expects
+					// packages are laid out in dependency order.
+					lib.DupTextSyms2 = append(lib.DupTextSyms2, sym.LoaderSym(gi))
+				}
+				continue // symbol in different object
+			}
+
+			lib.Textp2 = append(lib.Textp2, sym.LoaderSym(gi))
+		}
+	}
+
+	// Now redo the assignment of text symbols to libs/units.
+	for _, doInternal := range [2]bool{true, false} {
+		for idx, lib := range libs {
+			if intlibs[idx] != doInternal {
+				continue
+			}
+			libtextp2 := []sym.LoaderSym{}
+			tpls := [2][]sym.LoaderSym{lib.Textp2, lib.DupTextSyms2}
+			for _, textp2 := range tpls {
+				for _, s := range textp2 {
+					sym := Sym(s)
+					if l.attrReachable.has(sym) && !assignedToUnit.has(sym) {
+						libtextp2 = append(libtextp2, s)
+						unit := l.SymUnit(sym)
+						if unit != nil {
+							unit.Textp2 = append(unit.Textp2, s)
+							assignedToUnit.set(sym)
+						}
+					}
+				}
+			}
+			lib.Textp2 = libtextp2
+		}
+	}
+}
+
 // For debugging.
 func (l *Loader) Dump() {
 	fmt.Println("objs")
@@ -2333,7 +2458,7 @@ func (l *Loader) Dump() {
 	fmt.Println("extStart:", l.extStart)
 	fmt.Println("Nsyms:", len(l.objSyms))
 	fmt.Println("syms")
-	for i := Sym(1); i <= Sym(len(l.objSyms)); i++ {
+	for i := Sym(1); i < Sym(len(l.objSyms)); i++ {
 		pi := interface{}("")
 		if l.IsExternal(i) {
 			pi = fmt.Sprintf("<ext %d>", l.extIndex(i))

src/cmd/link/internal/sym/compilation_unit.go

@@ -6,6 +6,8 @@ package sym
 
 import "cmd/internal/dwarf"
 
+// LoaderSym holds a loader.Sym value. We can't refer to this
+// type from the sym package since loader imports sym.
 type LoaderSym int
 
 // CompilationUnit is an abstraction used by DWARF to represent a chunk of