[dev.regabi] cmd/compile,cmd/link: initial support for ABI wrappers

Add compiler support for emitting ABI wrappers by creating real IR as
opposed to introducing ABI aliases. At the moment these are "no-op"
wrappers in the sense that they make a simple call (using the existing
ABI) to their target. The assumption here is that once late call
expansion can handle both ABI0 and the "new" ABIInternal (register
version), it can expand the call to do the right thing.

Note that the runtime contains functions that do not strictly follow
the rules of the current Go ABI0; this has been handled in most cases
by treating these as ABIInternal instead (these changes have been made
in previous patches).

Generation of ABI wrappers (as opposed to ABI aliases) is currently
gated by GOEXPERIMENT=regabi -- wrapper generation is on by default if
GOEXPERIMENT=regabi is set and off otherwise (but can be turned on
using "-gcflags=all=-abiwrap -ldflags=-abiwrap"). Wrapper generation
currently only workd on AMD64; explicitly enabling wrapper for other
architectures (via the command line) is not supported.

Also in this patch are a few other command line options for debugging
(tracing and/or limiting wrapper creation). These will presumably go
away at some point.

Updates #27539, #40724.

Change-Id: I1ee3226fc15a3c32ca2087b8ef8e41dbe6df4a75
Reviewed-on: https://go-review.googlesource.com/c/go/+/270863
Run-TryBot: Than McIntosh <thanm@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Trust: Than McIntosh <thanm@google.com>

src/cmd/compile/internal/base/debug.go

@@ -51,6 +51,7 @@ type DebugFlags struct {
 	TypeAssert    int    `help:"print information about type assertion inlining"`
 	TypecheckInl  int    `help:"eager typechecking of inline function bodies"`
 	WB            int    `help:"print information about write barriers"`
+	ABIWrap       int    `help:"print information about ABI wrapper generation"`
 
 	any bool // set when any of the values have been set
 }

src/cmd/compile/internal/base/flag.go

@@ -81,6 +81,8 @@ type CmdFlags struct {
 	CompilingRuntime bool "flag:\"+\" help:\"compiling runtime\""
 
 	// Longer names
+	ABIWrap            bool         "help:\"enable generation of ABI wrappers\""
+	ABIWrapLimit       int          "help:\"emit at most N ABI wrappers (for debugging)\""
 	AsmHdr             string       "help:\"write assembly header to `file`\""
 	Bench              string       "help:\"append benchmark times to `file`\""
 	BlockProfile       string       "help:\"write block profile to `file`\""
@@ -140,6 +142,7 @@ func ParseFlags() {
 	Flag.LowerP = &Ctxt.Pkgpath
 	Flag.LowerV = &Ctxt.Debugvlog
 
+	Flag.ABIWrap = objabi.Regabi_enabled != 0
 	Flag.Dwarf = objabi.GOARCH != "wasm"
 	Flag.DwarfBASEntries = &Ctxt.UseBASEntries
 	Flag.DwarfLocationLists = &Ctxt.Flag_locationlists

src/cmd/compile/internal/gc/gsubr.go

@@ -34,9 +34,12 @@ import (
 	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/ssa"
+	"cmd/compile/internal/types"
 	"cmd/internal/obj"
 	"cmd/internal/objabi"
 	"cmd/internal/src"
+	"fmt"
+	"os"
 )
 
 var sharedProgArray = new([10000]obj.Prog) // *T instead of T to work around issue 19839
@@ -187,32 +190,154 @@ func (pp *Progs) settext(fn *ir.Func) {
 	ptxt.From.Sym = fn.LSym
 }
 
+// makeABIWrapper creates a new function that wraps a cross-ABI call
+// to "f".  The wrapper is marked as an ABIWRAPPER.
+func makeABIWrapper(f *ir.Func, wrapperABI obj.ABI) {
+
+	// Q: is this needed?
+	savepos := base.Pos
+	savedclcontext := dclcontext
+	savedcurfn := Curfn
+
+	base.Pos = autogeneratedPos
+	dclcontext = ir.PEXTERN
+
+	// At the moment we don't support wrapping a method, we'd need machinery
+	// below to handle the receiver. Panic if we see this scenario.
+	ft := f.Nname.Ntype.Type()
+	if ft.NumRecvs() != 0 {
+		panic("makeABIWrapper support for wrapping methods not implemented")
+	}
+
+	// Manufacture a new func type to use for the wrapper.
+	var noReceiver *ir.Field
+	tfn := ir.NewFuncType(base.Pos,
+		noReceiver,
+		structargs(ft.Params(), true),
+		structargs(ft.Results(), false))
+
+	// Reuse f's types.Sym to create a new ODCLFUNC/function.
+	fn := dclfunc(f.Nname.Sym(), tfn)
+	fn.SetDupok(true)
+	fn.SetWrapper(true) // ignore frame for panic+recover matching
+
+	// Select LSYM now.
+	asym := base.Ctxt.LookupABI(f.LSym.Name, wrapperABI)
+	asym.Type = objabi.STEXT
+	if fn.LSym != nil {
+		panic("unexpected")
+	}
+	fn.LSym = asym
+
+	// ABI0-to-ABIInternal wrappers will be mainly loading params from
+	// stack into registers (and/or storing stack locations back to
+	// registers after the wrapped call); in most cases they won't
+	// need to allocate stack space, so it should be OK to mark them
+	// as NOSPLIT in these cases. In addition, my assumption is that
+	// functions written in assembly are NOSPLIT in most (but not all)
+	// cases. In the case of an ABIInternal target that has too many
+	// parameters to fit into registers, the wrapper would need to
+	// allocate stack space, but this seems like an unlikely scenario.
+	// Hence: mark these wrappers NOSPLIT.
+	//
+	// ABIInternal-to-ABI0 wrappers on the other hand will be taking
+	// things in registers and pushing them onto the stack prior to
+	// the ABI0 call, meaning that they will always need to allocate
+	// stack space. If the compiler marks them as NOSPLIT this seems
+	// as though it could lead to situations where the the linker's
+	// nosplit-overflow analysis would trigger a link failure. On the
+	// other hand if they not tagged NOSPLIT then this could cause
+	// problems when building the runtime (since there may be calls to
+	// asm routine in cases where it's not safe to grow the stack). In
+	// most cases the wrapper would be (in effect) inlined, but are
+	// there (perhaps) indirect calls from the runtime that could run
+	// into trouble here.
+	// FIXME: at the moment all.bash does not pass when I leave out
+	// NOSPLIT for these wrappers, so all are currently tagged with NOSPLIT.
+	setupTextLSym(fn, obj.NOSPLIT|obj.ABIWRAPPER)
+
+	// Generate call. Use tail call if no params and no returns,
+	// but a regular call otherwise.
+	//
+	// Note: ideally we would be using a tail call in cases where
+	// there are params but no returns for ABI0->ABIInternal wrappers,
+	// provided that all params fit into registers (e.g. we don't have
+	// to allocate any stack space). Doing this will require some
+	// extra work in typecheck/walk/ssa, might want to add a new node
+	// OTAILCALL or something to this effect.
+	var call ir.Node
+	if tfn.Type().NumResults() == 0 && tfn.Type().NumParams() == 0 && tfn.Type().NumRecvs() == 0 {
+		call = nodSym(ir.ORETJMP, nil, f.Nname.Sym())
+	} else {
+		call = ir.Nod(ir.OCALL, f.Nname, nil)
+		call.PtrList().Set(paramNnames(tfn.Type()))
+		call.SetIsDDD(tfn.Type().IsVariadic())
+		if tfn.Type().NumResults() > 0 {
+			n := ir.Nod(ir.ORETURN, nil, nil)
+			n.PtrList().Set1(call)
+			call = n
+		}
+	}
+	fn.PtrBody().Append(call)
+
+	funcbody()
+	if base.Debug.DclStack != 0 {
+		testdclstack()
+	}
+
+	typecheckFunc(fn)
+	Curfn = fn
+	typecheckslice(fn.Body().Slice(), ctxStmt)
+
+	escapeFuncs([]*ir.Func{fn}, false)
+
+	Target.Decls = append(Target.Decls, fn)
+
+	// Restore previous context.
+	base.Pos = savepos
+	dclcontext = savedclcontext
+	Curfn = savedcurfn
+}
+
 // initLSym defines f's obj.LSym and initializes it based on the
 // properties of f. This includes setting the symbol flags and ABI and
 // creating and initializing related DWARF symbols.
 //
 // initLSym must be called exactly once per function and must be
 // called for both functions with bodies and functions without bodies.
+// For body-less functions, we only create the LSym; for functions
+// with bodies call a helper to setup up / populate the LSym.
 func initLSym(f *ir.Func, hasBody bool) {
+	// FIXME: for new-style ABI wrappers, we set up the lsym at the
+	// point the wrapper is created.
+	if f.LSym != nil && base.Flag.ABIWrap {
+		return
+	}
+	selectLSym(f, hasBody)
+	if hasBody {
+		setupTextLSym(f, 0)
+	}
+}
+
+// selectLSym sets up the LSym for a given function, and
+// makes calls to helpers to create ABI wrappers if needed.
+func selectLSym(f *ir.Func, hasBody bool) {
 	if f.LSym != nil {
 		base.Fatalf("Func.initLSym called twice")
 	}
 
 	if nam := f.Nname; !ir.IsBlank(nam) {
-		f.LSym = nam.Sym().Linksym()
-		if f.Pragma&ir.Systemstack != 0 {
-			f.LSym.Set(obj.AttrCFunc, true)
-		}
 
-		var aliasABI obj.ABI
+		var wrapperABI obj.ABI
-		needABIAlias := false
+		needABIWrapper := false
-		defABI, hasDefABI := symabiDefs[f.LSym.Name]
+		defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()]
 		if hasDefABI && defABI == obj.ABI0 {
 			// Symbol is defined as ABI0. Create an
 			// Internal -> ABI0 wrapper.
-			f.LSym.SetABI(obj.ABI0)
+			f.LSym = nam.Sym().LinksymABI0()
-			needABIAlias, aliasABI = true, obj.ABIInternal
+			needABIWrapper, wrapperABI = true, obj.ABIInternal
 		} else {
+			f.LSym = nam.Sym().Linksym()
 			// No ABI override. Check that the symbol is
 			// using the expected ABI.
 			want := obj.ABIInternal
@@ -220,6 +345,9 @@ func initLSym(f *ir.Func, hasBody bool) {
 				base.Fatalf("function symbol %s has the wrong ABI %v, expected %v", f.LSym.Name, f.LSym.ABI(), want)
 			}
 		}
+		if f.Pragma&ir.Systemstack != 0 {
+			f.LSym.Set(obj.AttrCFunc, true)
+		}
 
 		isLinknameExported := nam.Sym().Linkname != "" && (hasBody || hasDefABI)
 		if abi, ok := symabiRefs[f.LSym.Name]; (ok && abi == obj.ABI0) || isLinknameExported {
@@ -235,32 +363,39 @@ func initLSym(f *ir.Func, hasBody bool) {
 			// using linkname and we don't want to create
 			// duplicate ABI wrappers.
 			if f.LSym.ABI() != obj.ABI0 {
-				needABIAlias, aliasABI = true, obj.ABI0
+				needABIWrapper, wrapperABI = true, obj.ABI0
 			}
 		}
 
-		if needABIAlias {
+		if needABIWrapper {
-			// These LSyms have the same name as the
+			if !useABIWrapGen(f) {
-			// native function, so we create them directly
+				// Fallback: use alias instead. FIXME.
-			// rather than looking them up. The uniqueness
+
-			// of f.lsym ensures uniqueness of asym.
+				// These LSyms have the same name as the
-			asym := &obj.LSym{
+				// native function, so we create them directly
-				Name: f.LSym.Name,
+				// rather than looking them up. The uniqueness
-				Type: objabi.SABIALIAS,
+				// of f.lsym ensures uniqueness of asym.
-				R:    []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational"
+				asym := &obj.LSym{
+					Name: f.LSym.Name,
+					Type: objabi.SABIALIAS,
+					R:    []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational"
+				}
+				asym.SetABI(wrapperABI)
+				asym.Set(obj.AttrDuplicateOK, true)
+				base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym)
+			} else {
+				if base.Debug.ABIWrap != 0 {
+					fmt.Fprintf(os.Stderr, "=-= %v to %v wrapper for %s.%s\n",
+						wrapperABI, 1-wrapperABI, types.LocalPkg.Path, f.LSym.Name)
+				}
+				makeABIWrapper(f, wrapperABI)
 			}
-			asym.SetABI(aliasABI)
-			asym.Set(obj.AttrDuplicateOK, true)
-			base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym)
 		}
 	}
+}
 
-	if !hasBody {
+// setupTextLsym initializes the LSym for a with-body text symbol.
-		// For body-less functions, we only create the LSym.
+func setupTextLSym(f *ir.Func, flag int) {
-		return
-	}
-
-	var flag int
 	if f.Dupok() {
 		flag |= obj.DUPOK
 	}

src/cmd/compile/internal/gc/main.go

@@ -1144,3 +1144,26 @@ func initializeTypesPackage() {
 
 	initUniverse()
 }
+
+// useNewABIWrapGen returns TRUE if the compiler should generate an
+// ABI wrapper for the function 'f'.
+func useABIWrapGen(f *ir.Func) bool {
+	if !base.Flag.ABIWrap {
+		return false
+	}
+
+	// Support limit option for bisecting.
+	if base.Flag.ABIWrapLimit == 1 {
+		return false
+	}
+	if base.Flag.ABIWrapLimit < 1 {
+		return true
+	}
+	base.Flag.ABIWrapLimit--
+	if base.Debug.ABIWrap != 0 && base.Flag.ABIWrapLimit == 1 {
+		fmt.Fprintf(os.Stderr, "=-= limit reached after new wrapper for %s\n",
+			f.LSym.Name)
+	}
+
+	return true
+}

src/cmd/compile/internal/gc/pgen.go

@@ -32,7 +32,6 @@ func emitptrargsmap(fn *ir.Func) {
 		return
 	}
 	lsym := base.Ctxt.Lookup(fn.LSym.Name + ".args_stackmap")
-
 	nptr := int(fn.Type().ArgWidth() / int64(Widthptr))
 	bv := bvalloc(int32(nptr) * 2)
 	nbitmap := 1
@@ -399,7 +398,11 @@ func debuginfo(fnsym *obj.LSym, infosym *obj.LSym, curfn interface{}) ([]dwarf.S
 	fn := curfn.(*ir.Func)
 
 	if fn.Nname != nil {
-		if expect := fn.Sym().Linksym(); fnsym != expect {
+		expect := fn.Sym().Linksym()
+		if fnsym.ABI() == obj.ABI0 {
+			expect = fn.Sym().LinksymABI0()
+		}
+		if fnsym != expect {
 			base.Fatalf("unexpected fnsym: %v != %v", fnsym, expect)
 		}
 	}

src/cmd/compile/internal/gc/racewalk.go

@@ -61,7 +61,7 @@ func ispkgin(pkgs []string) bool {
 }
 
 func instrument(fn *ir.Func) {
-	if fn.Pragma&ir.Norace != 0 {
+	if fn.Pragma&ir.Norace != 0 || (fn.Sym().Linksym() != nil && fn.Sym().Linksym().ABIWrapper()) {
 		return
 	}
 

src/cmd/compile/internal/gc/ssa.go

@@ -1421,7 +1421,7 @@ func (s *state) stmt(n ir.Node) {
 	case ir.ORETJMP:
 		b := s.exit()
 		b.Kind = ssa.BlockRetJmp // override BlockRet
-		b.Aux = n.Sym().Linksym()
+		b.Aux = callTargetLSym(n.Sym(), s.curfn.LSym)
 
 	case ir.OCONTINUE, ir.OBREAK:
 		var to *ssa.Block
@@ -4826,11 +4826,11 @@ func (s *state) call(n *ir.CallExpr, k callKind, returnResultAddr bool) *ssa.Val
 			}
 		case sym != nil:
 			if testLateExpansion {
-				aux := ssa.StaticAuxCall(sym.Linksym(), ACArgs, ACResults)
+				aux := ssa.StaticAuxCall(callTargetLSym(sym, s.curfn.LSym), ACArgs, ACResults)
 				call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux)
 				call.AddArgs(callArgs...)
 			} else {
-				call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, ssa.StaticAuxCall(sym.Linksym(), ACArgs, ACResults), s.mem())
+				call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, ssa.StaticAuxCall(callTargetLSym(sym, s.curfn.LSym), ACArgs, ACResults), s.mem())
 			}
 		default:
 			s.Fatalf("bad call type %v %v", n.Op(), n)
@@ -7291,3 +7291,46 @@ func clobberBase(n ir.Node) ir.Node {
 	}
 	return n
 }
+
+// callTargetLSym determines the correct LSym for 'callee' when called
+// from function 'caller'. There are a couple of different scenarios
+// to contend with here:
+//
+// 1. if 'caller' is an ABI wrapper, then we always want to use the
+//    LSym from the Func for the callee.
+//
+// 2. if 'caller' is not an ABI wrapper, then we looked at the callee
+//    to see if it corresponds to a "known" ABI0 symbol (e.g. assembly
+//    routine defined in the current package); if so, we want the call to
+//    directly target the ABI0 symbol (effectively bypassing the
+//    ABIInternal->ABI0 wrapper for 'callee').
+//
+// 3. in all other cases, want the regular ABIInternal linksym
+//
+func callTargetLSym(callee *types.Sym, callerLSym *obj.LSym) *obj.LSym {
+	lsym := callee.Linksym()
+	if !base.Flag.ABIWrap {
+		return lsym
+	}
+	if ir.AsNode(callee.Def) == nil {
+		return lsym
+	}
+	ndclfunc := ir.AsNode(callee.Def).Name().Defn
+	if ndclfunc == nil {
+		return lsym
+	}
+	// check for case 1 above
+	if callerLSym.ABIWrapper() {
+		if nlsym := ndclfunc.Func().LSym; nlsym != nil {
+			lsym = nlsym
+		}
+	} else {
+		// check for case 2 above
+		nam := ndclfunc.Func().Nname
+		defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()]
+		if hasDefABI && defABI == obj.ABI0 {
+			lsym = nam.Sym().LinksymABI0()
+		}
+	}
+	return lsym
+}

src/cmd/compile/internal/types/sym.go

@@ -93,6 +93,23 @@ func (sym *Sym) Linksym() *obj.LSym {
 	return base.Ctxt.LookupInit(sym.LinksymName(), initPkg)
 }
 
+// LinksymABI0 looks up or creates an ABI0 linker symbol for "sym",
+// in cases where we want to specifically select the ABI0 version of
+// a symbol (typically used only for ABI wrappers).
+func (sym *Sym) LinksymABI0() *obj.LSym {
+	if sym == nil {
+		return nil
+	}
+	initPkg := func(r *obj.LSym) {
+		if sym.Linkname != "" {
+			r.Pkg = "_"
+		} else {
+			r.Pkg = sym.Pkg.Prefix
+		}
+	}
+	return base.Ctxt.LookupABIInit(sym.LinksymName(), obj.ABI0, initPkg)
+}
+
 // Less reports whether symbol a is ordered before symbol b.
 //
 // Symbols are ordered exported before non-exported, then by name, and

src/cmd/internal/obj/link.go

@@ -635,6 +635,10 @@ const (
 	// ContentAddressable indicates this is a content-addressable symbol.
 	AttrContentAddressable
 
+	// ABI wrapper is set for compiler-generated text symbols that
+	// convert between ABI0 and ABIInternal calling conventions.
+	AttrABIWrapper
+
 	// attrABIBase is the value at which the ABI is encoded in
 	// Attribute. This must be last; all bits after this are
 	// assumed to be an ABI value.
@@ -660,6 +664,7 @@ func (a Attribute) TopFrame() bool           { return a&AttrTopFrame != 0 }
 func (a Attribute) Indexed() bool            { return a&AttrIndexed != 0 }
 func (a Attribute) UsedInIface() bool        { return a&AttrUsedInIface != 0 }
 func (a Attribute) ContentAddressable() bool { return a&AttrContentAddressable != 0 }
+func (a Attribute) ABIWrapper() bool         { return a&AttrABIWrapper != 0 }
 
 func (a *Attribute) Set(flag Attribute, value bool) {
 	if value {
@@ -695,6 +700,7 @@ var textAttrStrings = [...]struct {
 	{bit: AttrTopFrame, s: "TOPFRAME"},
 	{bit: AttrIndexed, s: ""},
 	{bit: AttrContentAddressable, s: ""},
+	{bit: AttrABIWrapper, s: "ABIWRAPPER"},
 }
 
 // TextAttrString formats a for printing in as part of a TEXT prog.

src/cmd/internal/obj/plist.go

@@ -80,6 +80,11 @@ func Flushplist(ctxt *Link, plist *Plist, newprog ProgAlloc, myimportpath string
 		if !strings.HasPrefix(s.Name, "\"\".") {
 			continue
 		}
+		if s.ABIWrapper() {
+			// Don't create an args_stackmap symbol reference for an ABI
+			// wrapper function
+			continue
+		}
 		found := false
 		for p := s.Func().Text; p != nil; p = p.Link {
 			if p.As == AFUNCDATA && p.From.Type == TYPE_CONST && p.From.Offset == objabi.FUNCDATA_ArgsPointerMaps {
@@ -134,6 +139,7 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) {
 	s.Set(AttrNoSplit, flag&NOSPLIT != 0)
 	s.Set(AttrReflectMethod, flag&REFLECTMETHOD != 0)
 	s.Set(AttrWrapper, flag&WRAPPER != 0)
+	s.Set(AttrABIWrapper, flag&ABIWRAPPER != 0)
 	s.Set(AttrNeedCtxt, flag&NEEDCTXT != 0)
 	s.Set(AttrNoFrame, flag&NOFRAME != 0)
 	s.Set(AttrTopFrame, flag&TOPFRAME != 0)

src/cmd/internal/obj/textflag.go

@@ -51,4 +51,7 @@ const (
 	// Function is the top of the call stack. Call stack unwinders should stop
 	// at this function.
 	TOPFRAME = 2048
+
+	// Function is an ABI wrapper.
+	ABIWRAPPER = 4096
 )

src/cmd/internal/obj/x86/obj6.go

@@ -637,7 +637,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
 		}
 	}
 
-	if !p.From.Sym.NoSplit() || p.From.Sym.Wrapper() {
+	if !p.From.Sym.NoSplit() || (p.From.Sym.Wrapper() && !p.From.Sym.ABIWrapper()) {
 		p = obj.Appendp(p, newprog)
 		p = load_g_cx(ctxt, p, newprog) // load g into CX
 	}
@@ -690,7 +690,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
 		p.To.Reg = REG_BP
 	}
 
-	if cursym.Func().Text.From.Sym.Wrapper() {
+	if cursym.Func().Text.From.Sym.Wrapper() && !cursym.Func().Text.From.Sym.ABIWrapper() {
 		// if g._panic != nil && g._panic.argp == FP {
 		//   g._panic.argp = bottom-of-frame
 		// }

src/cmd/link/internal/ld/main.go

@@ -92,11 +92,10 @@ var (
 	FlagRound         = flag.Int("R", -1, "set address rounding `quantum`")
 	FlagTextAddr      = flag.Int64("T", -1, "set text segment `address`")
 	flagEntrySymbol   = flag.String("E", "", "set `entry` symbol name")
-
+	cpuprofile        = flag.String("cpuprofile", "", "write cpu profile to `file`")
-	cpuprofile     = flag.String("cpuprofile", "", "write cpu profile to `file`")
+	memprofile        = flag.String("memprofile", "", "write memory profile to `file`")
-	memprofile     = flag.String("memprofile", "", "write memory profile to `file`")
+	memprofilerate    = flag.Int64("memprofilerate", 0, "set runtime.MemProfileRate to `rate`")
-	memprofilerate = flag.Int64("memprofilerate", 0, "set runtime.MemProfileRate to `rate`")
+	flagAbiWrap       = false
-
 	benchmarkFlag     = flag.String("benchmark", "", "set to 'mem' or 'cpu' to enable phase benchmarking")
 	benchmarkFileFlag = flag.String("benchmarkprofile", "", "emit phase profiles to `base`_phase.{cpu,mem}prof")
 )
@@ -135,6 +134,9 @@ func Main(arch *sys.Arch, theArch Arch) {
 	objabi.Flagfn1("X", "add string value `definition` of the form importpath.name=value", func(s string) { addstrdata1(ctxt, s) })
 	objabi.Flagcount("v", "print link trace", &ctxt.Debugvlog)
 	objabi.Flagfn1("importcfg", "read import configuration from `file`", ctxt.readImportCfg)
+	if objabi.Regabi_enabled != 0 {
+		flag.BoolVar(&flagAbiWrap, "abiwrap", true, "support ABI wrapper functions")
+	}
 
 	objabi.Flagparse(usage)
 

src/cmd/link/internal/ld/symtab.go

@@ -102,6 +102,41 @@ func putelfsym(ctxt *Link, x loader.Sym, typ elf.SymType, curbind elf.SymBind) {
 		elfshnum = xosect.Elfsect.(*ElfShdr).shnum
 	}
 
+	sname := ldr.SymExtname(x)
+
+	// For functions with ABI wrappers, we have to make sure that we
+	// don't wind up with two elf symbol table entries with the same
+	// name (since this will generated an error from the external
+	// linker). In the CgoExportStatic case, we want the ABI0 symbol
+	// to have the primary symbol table entry (since it's going to be
+	// called from C), so we rename the ABIInternal symbol. In all
+	// other cases, we rename the ABI0 symbol, since we want
+	// cross-load-module calls to target ABIInternal.
+	//
+	// TODO: generalize this for non-ELF (put the rename code in the
+	// loader, and store the rename result in SymExtname).
+	//
+	// TODO: avoid the ldr.Lookup calls below by instead using an aux
+	// sym or marker relocation to associate the wrapper with the
+	// wrapped function.
+	//
+	if flagAbiWrap {
+		if !ldr.IsExternal(x) && ldr.SymType(x) == sym.STEXT {
+			// First case
+			if ldr.SymVersion(x) == sym.SymVerABIInternal {
+				if s2 := ldr.Lookup(sname, sym.SymVerABI0); s2 != 0 && ldr.AttrCgoExportStatic(s2) && ldr.SymType(s2) == sym.STEXT {
+					sname = sname + ".abiinternal"
+				}
+			}
+			// Second case
+			if ldr.SymVersion(x) == sym.SymVerABI0 && !ldr.AttrCgoExportStatic(x) {
+				if s2 := ldr.Lookup(sname, sym.SymVerABIInternal); s2 != 0 && ldr.SymType(s2) == sym.STEXT {
+					sname = sname + ".abi0"
+				}
+			}
+		}
+	}
+
 	// One pass for each binding: elf.STB_LOCAL, elf.STB_GLOBAL,
 	// maybe one day elf.STB_WEAK.
 	bind := elf.STB_GLOBAL
@@ -140,8 +175,6 @@ func putelfsym(ctxt *Link, x loader.Sym, typ elf.SymType, curbind elf.SymBind) {
 		other |= 3 << 5
 	}
 
-	sname := ldr.SymExtname(x)
-
 	// When dynamically linking, we create Symbols by reading the names from
 	// the symbol tables of the shared libraries and so the names need to
 	// match exactly. Tools like DTrace will have to wait for now.

src/runtime/textflag.h

@@ -35,3 +35,5 @@
 // Function is the top of the call stack. Call stack unwinders should stop
 // at this function.
 #define TOPFRAME 2048
+// Function is an ABI wrapper.
+#define ABIWRAPPER 4096

test/nosplit.go

@@ -353,7 +353,14 @@ TestCases:
 			log.Fatal(err)
 		}
 
-		cmd := exec.Command("go", "build")
+		// Turn off ABI0 wrapper generation for now. The problem here is
+		// that in these test cases main.main is an assembly routine,
+		// thus calls to it will have to go through an ABI wrapper. The
+		// ABI wrapper will consume some stack space, which throws off
+		// the numbers.
+		workaround := "-gcflags=-abiwrap=0"
+
+		cmd := exec.Command("go", "build", workaround)
 		cmd.Dir = dir
 		output, err := cmd.CombinedOutput()
 		if err == nil {