reflect,runtime: use internal ABI for selected ASM routines, attempt 2

[This is a roll-forward of CL 262319, with a fix for some Darwin test
failures].

Change the definitions of selected runtime assembly routines
from ABI0 (the default) to ABIInternal. The ABIInternal def is
intended to indicate that these functions don't follow the existing Go
runtime ABI. In addition, convert the assembly reference to
runtime.main (from runtime.mainPC) to ABIInternal. Finally, for
functions such as "runtime.duffzero" that are called directly from
generated code, make sure that the compiler looks up the correct
ABI version.

This is intended to support the register abi work, however these
changes should not have any issues even when GOEXPERIMENT=regabi is
not in effect.

Updates #27539, #40724.

Change-Id: Idf507f1c06176073563845239e1a54dad51a9ea9
Reviewed-on: https://go-review.googlesource.com/c/go/+/266638
Trust: Than McIntosh <thanm@google.com>
Run-TryBot: Than McIntosh <thanm@google.com>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
This commit is contained in:
Than McIntosh 2020-10-14 08:36:11 -04:00
parent 1af388f1c0
commit a313eec386
20 changed files with 156 additions and 114 deletions

View file

@ -72,9 +72,9 @@ func initssaconfig() {
deferproc = sysfunc("deferproc") deferproc = sysfunc("deferproc")
deferprocStack = sysfunc("deferprocStack") deferprocStack = sysfunc("deferprocStack")
Deferreturn = sysfunc("deferreturn") Deferreturn = sysfunc("deferreturn")
Duffcopy = sysvar("duffcopy") // asm func with special ABI Duffcopy = sysfunc("duffcopy")
Duffzero = sysvar("duffzero") // asm func with special ABI Duffzero = sysfunc("duffzero")
gcWriteBarrier = sysvar("gcWriteBarrier") // asm func with special ABI gcWriteBarrier = sysfunc("gcWriteBarrier")
goschedguarded = sysfunc("goschedguarded") goschedguarded = sysfunc("goschedguarded")
growslice = sysfunc("growslice") growslice = sysfunc("growslice")
msanread = sysfunc("msanread") msanread = sysfunc("msanread")
@ -105,51 +105,51 @@ func initssaconfig() {
// asm funcs with special ABI // asm funcs with special ABI
if thearch.LinkArch.Name == "amd64" { if thearch.LinkArch.Name == "amd64" {
GCWriteBarrierReg = map[int16]*obj.LSym{ GCWriteBarrierReg = map[int16]*obj.LSym{
x86.REG_AX: sysvar("gcWriteBarrier"), x86.REG_AX: sysfunc("gcWriteBarrier"),
x86.REG_CX: sysvar("gcWriteBarrierCX"), x86.REG_CX: sysfunc("gcWriteBarrierCX"),
x86.REG_DX: sysvar("gcWriteBarrierDX"), x86.REG_DX: sysfunc("gcWriteBarrierDX"),
x86.REG_BX: sysvar("gcWriteBarrierBX"), x86.REG_BX: sysfunc("gcWriteBarrierBX"),
x86.REG_BP: sysvar("gcWriteBarrierBP"), x86.REG_BP: sysfunc("gcWriteBarrierBP"),
x86.REG_SI: sysvar("gcWriteBarrierSI"), x86.REG_SI: sysfunc("gcWriteBarrierSI"),
x86.REG_R8: sysvar("gcWriteBarrierR8"), x86.REG_R8: sysfunc("gcWriteBarrierR8"),
x86.REG_R9: sysvar("gcWriteBarrierR9"), x86.REG_R9: sysfunc("gcWriteBarrierR9"),
} }
} }
if thearch.LinkArch.Family == sys.Wasm { if thearch.LinkArch.Family == sys.Wasm {
BoundsCheckFunc[ssa.BoundsIndex] = sysvar("goPanicIndex") BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("goPanicIndex")
BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("goPanicIndexU") BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("goPanicIndexU")
BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("goPanicSliceAlen") BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("goPanicSliceAlen")
BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("goPanicSliceAlenU") BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("goPanicSliceAlenU")
BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("goPanicSliceAcap") BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("goPanicSliceAcap")
BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("goPanicSliceAcapU") BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("goPanicSliceAcapU")
BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("goPanicSliceB") BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("goPanicSliceB")
BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("goPanicSliceBU") BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("goPanicSliceBU")
BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("goPanicSlice3Alen") BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("goPanicSlice3Alen")
BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("goPanicSlice3AlenU") BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("goPanicSlice3AlenU")
BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("goPanicSlice3Acap") BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("goPanicSlice3Acap")
BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("goPanicSlice3AcapU") BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("goPanicSlice3AcapU")
BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("goPanicSlice3B") BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("goPanicSlice3B")
BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("goPanicSlice3BU") BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("goPanicSlice3BU")
BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("goPanicSlice3C") BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("goPanicSlice3C")
BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("goPanicSlice3CU") BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("goPanicSlice3CU")
} else { } else {
BoundsCheckFunc[ssa.BoundsIndex] = sysvar("panicIndex") BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("panicIndex")
BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("panicIndexU") BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("panicIndexU")
BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("panicSliceAlen") BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("panicSliceAlen")
BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("panicSliceAlenU") BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("panicSliceAlenU")
BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("panicSliceAcap") BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("panicSliceAcap")
BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("panicSliceAcapU") BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("panicSliceAcapU")
BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("panicSliceB") BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("panicSliceB")
BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("panicSliceBU") BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("panicSliceBU")
BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("panicSlice3Alen") BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("panicSlice3Alen")
BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("panicSlice3AlenU") BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("panicSlice3AlenU")
BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("panicSlice3Acap") BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("panicSlice3Acap")
BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("panicSlice3AcapU") BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("panicSlice3AcapU")
BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("panicSlice3B") BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("panicSlice3B")
BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("panicSlice3BU") BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("panicSlice3BU")
BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("panicSlice3C") BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("panicSlice3C")
BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("panicSlice3CU") BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("panicSlice3CU")
} }
if thearch.LinkArch.PtrSize == 4 { if thearch.LinkArch.PtrSize == 4 {
ExtendCheckFunc[ssa.BoundsIndex] = sysvar("panicExtendIndex") ExtendCheckFunc[ssa.BoundsIndex] = sysvar("panicExtendIndex")

View file

@ -129,7 +129,6 @@ var (
morestackNoCtxt *obj.LSym morestackNoCtxt *obj.LSym
gcWriteBarrier *obj.LSym gcWriteBarrier *obj.LSym
sigpanic *obj.LSym sigpanic *obj.LSym
sigpanic0 *obj.LSym
deferreturn *obj.LSym deferreturn *obj.LSym
jmpdefer *obj.LSym jmpdefer *obj.LSym
) )
@ -142,9 +141,8 @@ const (
func instinit(ctxt *obj.Link) { func instinit(ctxt *obj.Link) {
morestack = ctxt.Lookup("runtime.morestack") morestack = ctxt.Lookup("runtime.morestack")
morestackNoCtxt = ctxt.Lookup("runtime.morestack_noctxt") morestackNoCtxt = ctxt.Lookup("runtime.morestack_noctxt")
gcWriteBarrier = ctxt.Lookup("runtime.gcWriteBarrier") gcWriteBarrier = ctxt.LookupABI("runtime.gcWriteBarrier", obj.ABIInternal)
sigpanic = ctxt.LookupABI("runtime.sigpanic", obj.ABIInternal) sigpanic = ctxt.LookupABI("runtime.sigpanic", obj.ABIInternal)
sigpanic0 = ctxt.LookupABI("runtime.sigpanic", 0) // sigpanic called from assembly, which has ABI0
deferreturn = ctxt.LookupABI("runtime.deferreturn", obj.ABIInternal) deferreturn = ctxt.LookupABI("runtime.deferreturn", obj.ABIInternal)
// jmpdefer is defined in assembly as ABI0, but what we're // jmpdefer is defined in assembly as ABI0, but what we're
// looking for is the *call* to jmpdefer from the Go function // looking for is the *call* to jmpdefer from the Go function
@ -493,7 +491,7 @@ func preprocess(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) {
} }
// return value of call is on the top of the stack, indicating whether to unwind the WebAssembly stack // return value of call is on the top of the stack, indicating whether to unwind the WebAssembly stack
if call.As == ACALLNORESUME && call.To.Sym != sigpanic && call.To.Sym != sigpanic0 { // sigpanic unwinds the stack, but it never resumes if call.As == ACALLNORESUME && call.To.Sym != sigpanic { // sigpanic unwinds the stack, but it never resumes
// trying to unwind WebAssembly stack but call has no resume point, terminate with error // trying to unwind WebAssembly stack but call has no resume point, terminate with error
p = appendp(p, AIf) p = appendp(p, AIf)
p = appendp(p, obj.AUNDEF) p = appendp(p, obj.AUNDEF)

View file

@ -324,9 +324,9 @@ func rewriteToUseGot(ctxt *obj.Link, p *obj.Prog, newprog obj.ProgAlloc) {
// flags and duffzero on 386 does not otherwise do so). // flags and duffzero on 386 does not otherwise do so).
var sym *obj.LSym var sym *obj.LSym
if p.As == obj.ADUFFZERO { if p.As == obj.ADUFFZERO {
sym = ctxt.Lookup("runtime.duffzero") sym = ctxt.LookupABI("runtime.duffzero", obj.ABIInternal)
} else { } else {
sym = ctxt.Lookup("runtime.duffcopy") sym = ctxt.LookupABI("runtime.duffcopy", obj.ABIInternal)
} }
offset := p.To.Offset offset := p.To.Offset
p.As = mov p.As = mov

View file

@ -9,7 +9,9 @@
// See the comment on the declaration of makeFuncStub in makefunc.go // See the comment on the declaration of makeFuncStub in makefunc.go
// for more details. // for more details.
// No arg size here; runtime pulls arg map out of the func value. // No arg size here; runtime pulls arg map out of the func value.
TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 // makeFuncStub must be ABIInternal because it is placed directly
// in function values.
TEXT ·makeFuncStub<ABIInternal>(SB),(NOSPLIT|WRAPPER),$32
NO_LOCAL_POINTERS NO_LOCAL_POINTERS
MOVQ DX, 0(SP) MOVQ DX, 0(SP)
LEAQ argframe+0(FP), CX LEAQ argframe+0(FP), CX
@ -17,14 +19,16 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32
MOVB $0, 24(SP) MOVB $0, 24(SP)
LEAQ 24(SP), AX LEAQ 24(SP), AX
MOVQ AX, 16(SP) MOVQ AX, 16(SP)
CALL ·callReflect(SB) CALL ·callReflect<ABIInternal>(SB)
RET RET
// methodValueCall is the code half of the function returned by makeMethodValue. // methodValueCall is the code half of the function returned by makeMethodValue.
// See the comment on the declaration of methodValueCall in makefunc.go // See the comment on the declaration of methodValueCall in makefunc.go
// for more details. // for more details.
// No arg size here; runtime pulls arg map out of the func value. // No arg size here; runtime pulls arg map out of the func value.
TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 // methodValueCall must be ABIInternal because it is placed directly
// in function values.
TEXT ·methodValueCall<ABIInternal>(SB),(NOSPLIT|WRAPPER),$32
NO_LOCAL_POINTERS NO_LOCAL_POINTERS
MOVQ DX, 0(SP) MOVQ DX, 0(SP)
LEAQ argframe+0(FP), CX LEAQ argframe+0(FP), CX
@ -32,5 +36,5 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32
MOVB $0, 24(SP) MOVB $0, 24(SP)
LEAQ 24(SP), AX LEAQ 24(SP), AX
MOVQ AX, 16(SP) MOVQ AX, 16(SP)
CALL ·callMethod(SB) CALL ·callMethod<ABIInternal>(SB)
RET RET

View file

@ -84,7 +84,9 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
DATA _rt0_amd64_lib_argv<>(SB)/8, $0 DATA _rt0_amd64_lib_argv<>(SB)/8, $0
GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8 GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
TEXT runtime·rt0_go(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stack-based Go ABI (and
// in addition there are no calls to this entry point from Go code).
TEXT runtime·rt0_go<ABIInternal>(SB),NOSPLIT,$0
// copy arguments forward on an even stack // copy arguments forward on an even stack
MOVQ DI, AX // argc MOVQ DI, AX // argc
MOVQ SI, BX // argv MOVQ SI, BX // argv
@ -229,10 +231,13 @@ ok:
// Prevent dead-code elimination of debugCallV1, which is // Prevent dead-code elimination of debugCallV1, which is
// intended to be called by debuggers. // intended to be called by debuggers.
MOVQ $runtime·debugCallV1(SB), AX MOVQ $runtime·debugCallV1<ABIInternal>(SB), AX
RET RET
DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) // mainPC is a function value for runtime.main, to be passed to newproc.
// The reference to runtime.main is made via ABIInternal, since the
// actual function (not the ABI0 wrapper) is needed by newproc.
DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
GLOBL runtime·mainPC(SB),RODATA,$8 GLOBL runtime·mainPC(SB),RODATA,$8
TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
@ -468,7 +473,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
JMP AX JMP AX
// Note: can't just "JMP NAME(SB)" - bad inlining results. // Note: can't just "JMP NAME(SB)" - bad inlining results.
TEXT ·reflectcall(SB), NOSPLIT, $0-32 TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
MOVLQZX argsize+24(FP), CX MOVLQZX argsize+24(FP), CX
DISPATCH(runtime·call16, 16) DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32) DISPATCH(runtime·call32, 32)
@ -1354,8 +1359,11 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0
RET RET
// The top-most function running on a goroutine // The top-most function running on a goroutine
// returns to goexit+PCQuantum. // returns to goexit+PCQuantum. Defined as ABIInternal
TEXT runtime·goexit(SB),NOSPLIT,$0-0 // so as to make it identifiable to traceback (this
// function it used as a sentinel; traceback wants to
// see the func PC, not a wrapper PC).
TEXT runtime·goexit<ABIInternal>(SB),NOSPLIT,$0-0
BYTE $0x90 // NOP BYTE $0x90 // NOP
CALL runtime·goexit1(SB) // does not return CALL runtime·goexit1(SB) // does not return
// traceback from goexit1 must hit code range of goexit // traceback from goexit1 must hit code range of goexit
@ -1377,7 +1385,8 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
// - AX is the value being written at DI // - AX is the value being written at DI
// It clobbers FLAGS. It does not clobber any general-purpose registers, // It clobbers FLAGS. It does not clobber any general-purpose registers,
// but may clobber others (e.g., SSE registers). // but may clobber others (e.g., SSE registers).
TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 // Defined as ABIInternal since it does not use the stack-based Go ABI.
TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$120
// Save the registers clobbered by the fast path. This is slightly // Save the registers clobbered by the fast path. This is slightly
// faster than having the caller spill these. // faster than having the caller spill these.
MOVQ R14, 104(SP) MOVQ R14, 104(SP)
@ -1461,51 +1470,58 @@ flush:
JMP ret JMP ret
// gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX. // gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX.
TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierCX<ABIInternal>(SB),NOSPLIT,$0
XCHGQ CX, AX XCHGQ CX, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ CX, AX XCHGQ CX, AX
RET RET
// gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX. // gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX.
TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierDX<ABIInternal>(SB),NOSPLIT,$0
XCHGQ DX, AX XCHGQ DX, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ DX, AX XCHGQ DX, AX
RET RET
// gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX. // gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX.
TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierBX<ABIInternal>(SB),NOSPLIT,$0
XCHGQ BX, AX XCHGQ BX, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ BX, AX XCHGQ BX, AX
RET RET
// gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP. // gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP.
TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierBP<ABIInternal>(SB),NOSPLIT,$0
XCHGQ BP, AX XCHGQ BP, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ BP, AX XCHGQ BP, AX
RET RET
// gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI. // gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI.
TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierSI<ABIInternal>(SB),NOSPLIT,$0
XCHGQ SI, AX XCHGQ SI, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ SI, AX XCHGQ SI, AX
RET RET
// gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8. // gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8.
TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierR8<ABIInternal>(SB),NOSPLIT,$0
XCHGQ R8, AX XCHGQ R8, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ R8, AX XCHGQ R8, AX
RET RET
// gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9. // gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9.
TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stable Go ABI.
TEXT runtime·gcWriteBarrierR9<ABIInternal>(SB),NOSPLIT,$0
XCHGQ R9, AX XCHGQ R9, AX
CALL runtime·gcWriteBarrier(SB) CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ R9, AX XCHGQ R9, AX
RET RET
@ -1544,7 +1560,10 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
// obey escape analysis requirements. Specifically, it must not pass // obey escape analysis requirements. Specifically, it must not pass
// a stack pointer to an escaping argument. debugCallV1 cannot check // a stack pointer to an escaping argument. debugCallV1 cannot check
// this invariant. // this invariant.
TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 //
// This is ABIInternal because Go code injects its PC directly into new
// goroutine stacks.
TEXT runtime·debugCallV1<ABIInternal>(SB),NOSPLIT,$152-0
// Save all registers that may contain pointers so they can be // Save all registers that may contain pointers so they can be
// conservatively scanned. // conservatively scanned.
// //
@ -1705,67 +1724,68 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
// in the caller's stack frame. These stubs write the args into that stack space and // in the caller's stack frame. These stubs write the args into that stack space and
// then tail call to the corresponding runtime handler. // then tail call to the corresponding runtime handler.
// The tail call makes these stubs disappear in backtraces. // The tail call makes these stubs disappear in backtraces.
TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 // Defined as ABIInternal since they do not use the stack-based Go ABI.
TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP) MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP) MOVQ CX, y+8(FP)
JMP runtime·goPanicIndex(SB) JMP runtime·goPanicIndex(SB)
TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP) MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP) MOVQ CX, y+8(FP)
JMP runtime·goPanicIndexU(SB) JMP runtime·goPanicIndexU(SB)
TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP) MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP) MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAlen(SB) JMP runtime·goPanicSliceAlen(SB)
TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP) MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP) MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAlenU(SB) JMP runtime·goPanicSliceAlenU(SB)
TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP) MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP) MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAcap(SB) JMP runtime·goPanicSliceAcap(SB)
TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP) MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP) MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAcapU(SB) JMP runtime·goPanicSliceAcapU(SB)
TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP) MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP) MOVQ CX, y+8(FP)
JMP runtime·goPanicSliceB(SB) JMP runtime·goPanicSliceB(SB)
TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP) MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP) MOVQ CX, y+8(FP)
JMP runtime·goPanicSliceBU(SB) JMP runtime·goPanicSliceBU(SB)
TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP) MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP) MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3Alen(SB) JMP runtime·goPanicSlice3Alen(SB)
TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP) MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP) MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3AlenU(SB) JMP runtime·goPanicSlice3AlenU(SB)
TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP) MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP) MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3Acap(SB) JMP runtime·goPanicSlice3Acap(SB)
TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP) MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP) MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3AcapU(SB) JMP runtime·goPanicSlice3AcapU(SB)
TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP) MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP) MOVQ DX, y+8(FP)
JMP runtime·goPanicSlice3B(SB) JMP runtime·goPanicSlice3B(SB)
TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP) MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP) MOVQ DX, y+8(FP)
JMP runtime·goPanicSlice3BU(SB) JMP runtime·goPanicSlice3BU(SB)
TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP) MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP) MOVQ CX, y+8(FP)
JMP runtime·goPanicSlice3C(SB) JMP runtime·goPanicSlice3C(SB)
TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP) MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP) MOVQ CX, y+8(FP)
JMP runtime·goPanicSlice3CU(SB) JMP runtime·goPanicSlice3CU(SB)

View file

@ -196,7 +196,7 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16
Get CTXT Get CTXT
I64Eqz I64Eqz
If If
CALLNORESUME runtime·sigpanic(SB) CALLNORESUME runtime·sigpanic<ABIInternal>(SB)
End End
// caller sp after CALL // caller sp after CALL
@ -300,7 +300,7 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-32
I64Load fn+8(FP) I64Load fn+8(FP)
I64Eqz I64Eqz
If If
CALLNORESUME runtime·sigpanic(SB) CALLNORESUME runtime·sigpanic<ABIInternal>(SB)
End End
MOVW argsize+24(FP), R0 MOVW argsize+24(FP), R0

View file

@ -4,7 +4,7 @@
#include "textflag.h" #include "textflag.h"
TEXT runtime·duffzero(SB), NOSPLIT, $0-0 TEXT runtime·duffzero<ABIInternal>(SB), NOSPLIT, $0-0
MOVUPS X0,(DI) MOVUPS X0,(DI)
MOVUPS X0,16(DI) MOVUPS X0,16(DI)
MOVUPS X0,32(DI) MOVUPS X0,32(DI)
@ -103,7 +103,7 @@ TEXT runtime·duffzero(SB), NOSPLIT, $0-0
RET RET
TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 TEXT runtime·duffcopy<ABIInternal>(SB), NOSPLIT, $0-0
MOVUPS (SI), X0 MOVUPS (SI), X0
ADDQ $16, SI ADDQ $16, SI
MOVUPS X0, (DI) MOVUPS X0, (DI)

View file

@ -126,7 +126,8 @@ func header(arch string) {
} }
fmt.Fprintf(out, "#include \"go_asm.h\"\n") fmt.Fprintf(out, "#include \"go_asm.h\"\n")
fmt.Fprintf(out, "#include \"textflag.h\"\n\n") fmt.Fprintf(out, "#include \"textflag.h\"\n\n")
fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") fmt.Fprintf(out, "// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.\n")
fmt.Fprintf(out, "TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0\n")
} }
func p(f string, args ...interface{}) { func p(f string, args ...interface{}) {

View file

@ -3,7 +3,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
PUSHFL PUSHFL
ADJSP $156 ADJSP $156
NOP SP NOP SP

View file

@ -3,7 +3,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
PUSHQ BP PUSHQ BP
MOVQ SP, BP MOVQ SP, BP
// Save flags before clobbering them // Save flags before clobbering them

View file

@ -3,7 +3,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
MOVW.W R14, -188(R13) MOVW.W R14, -188(R13)
MOVW R0, 4(R13) MOVW R0, 4(R13)
MOVW R1, 8(R13) MOVW R1, 8(R13)

View file

@ -3,7 +3,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
MOVD R30, -496(RSP) MOVD R30, -496(RSP)
SUB $496, RSP SUB $496, RSP
#ifdef GOOS_linux #ifdef GOOS_linux

View file

@ -5,7 +5,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
MOVV R31, -488(R29) MOVV R31, -488(R29)
SUBV $488, R29 SUBV $488, R29
MOVV R1, 8(R29) MOVV R1, 8(R29)

View file

@ -5,7 +5,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
MOVW R31, -244(R29) MOVW R31, -244(R29)
SUB $244, R29 SUB $244, R29
MOVW R1, 4(R29) MOVW R1, 4(R29)

View file

@ -5,7 +5,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
MOVD R31, -488(R1) MOVD R31, -488(R1)
MOVD LR, R31 MOVD LR, R31
MOVDU R31, -520(R1) MOVDU R31, -520(R1)

View file

@ -3,7 +3,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
MOV X1, -472(X2) MOV X1, -472(X2)
ADD $-472, X2 ADD $-472, X2
MOV X3, 8(X2) MOV X3, 8(X2)

View file

@ -3,7 +3,8 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
IPM R10 IPM R10
MOVD R14, -248(R15) MOVD R14, -248(R15)
ADD $-248, R15 ADD $-248, R15

View file

@ -3,6 +3,7 @@
#include "go_asm.h" #include "go_asm.h"
#include "textflag.h" #include "textflag.h"
TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.
TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
// No async preemption on wasm // No async preemption on wasm
UNDEF UNDEF

View file

@ -41,7 +41,9 @@
// func runtime·raceread(addr uintptr) // func runtime·raceread(addr uintptr)
// Called from instrumented code. // Called from instrumented code.
TEXT runtime·raceread(SB), NOSPLIT, $0-8 // Defined as ABIInternal so as to avoid introducing a wrapper,
// which would render runtime.getcallerpc ineffective.
TEXT runtime·raceread<ABIInternal>(SB), NOSPLIT, $0-8
MOVQ addr+0(FP), RARG1 MOVQ addr+0(FP), RARG1
MOVQ (SP), RARG2 MOVQ (SP), RARG2
// void __tsan_read(ThreadState *thr, void *addr, void *pc); // void __tsan_read(ThreadState *thr, void *addr, void *pc);
@ -65,7 +67,9 @@ TEXT runtime·racereadpc(SB), NOSPLIT, $0-24
// func runtime·racewrite(addr uintptr) // func runtime·racewrite(addr uintptr)
// Called from instrumented code. // Called from instrumented code.
TEXT runtime·racewrite(SB), NOSPLIT, $0-8 // Defined as ABIInternal so as to avoid introducing a wrapper,
// which would render runtime.getcallerpc ineffective.
TEXT runtime·racewrite<ABIInternal>(SB), NOSPLIT, $0-8
MOVQ addr+0(FP), RARG1 MOVQ addr+0(FP), RARG1
MOVQ (SP), RARG2 MOVQ (SP), RARG2
// void __tsan_write(ThreadState *thr, void *addr, void *pc); // void __tsan_write(ThreadState *thr, void *addr, void *pc);
@ -114,7 +118,9 @@ TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24
// func runtime·racewriterange(addr, size uintptr) // func runtime·racewriterange(addr, size uintptr)
// Called from instrumented code. // Called from instrumented code.
TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 // Defined as ABIInternal so as to avoid introducing a wrapper,
// which would render runtime.getcallerpc ineffective.
TEXT runtime·racewriterange<ABIInternal>(SB), NOSPLIT, $0-16
MOVQ addr+0(FP), RARG1 MOVQ addr+0(FP), RARG1
MOVQ size+8(FP), RARG2 MOVQ size+8(FP), RARG2
MOVQ (SP), RARG3 MOVQ (SP), RARG3

View file

@ -380,7 +380,8 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32
POPQ BP POPQ BP
RET RET
TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Defined as ABIInternal since it does not use the stack-based Go ABI.
TEXT runtime·sigtramp<ABIInternal>(SB),NOSPLIT,$72
// Save callee-saved C registers, since the caller may be a C signal handler. // Save callee-saved C registers, since the caller may be a C signal handler.
MOVQ BX, bx-8(SP) MOVQ BX, bx-8(SP)
MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set
@ -407,7 +408,8 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$72
// Used instead of sigtramp in programs that use cgo. // Used instead of sigtramp in programs that use cgo.
// Arguments from kernel are in DI, SI, DX. // Arguments from kernel are in DI, SI, DX.
TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stack-based Go ABI.
TEXT runtime·cgoSigtramp<ABIInternal>(SB),NOSPLIT,$0
// If no traceback function, do usual sigtramp. // If no traceback function, do usual sigtramp.
MOVQ runtime·cgoTraceback(SB), AX MOVQ runtime·cgoTraceback(SB), AX
TESTQ AX, AX TESTQ AX, AX
@ -450,12 +452,12 @@ TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0
// The first three arguments, and the fifth, are already in registers. // The first three arguments, and the fifth, are already in registers.
// Set the two remaining arguments now. // Set the two remaining arguments now.
MOVQ runtime·cgoTraceback(SB), CX MOVQ runtime·cgoTraceback(SB), CX
MOVQ $runtime·sigtramp(SB), R9 MOVQ $runtime·sigtramp<ABIInternal>(SB), R9
MOVQ _cgo_callers(SB), AX MOVQ _cgo_callers(SB), AX
JMP AX JMP AX
sigtramp: sigtramp:
JMP runtime·sigtramp(SB) JMP runtime·sigtramp<ABIInternal>(SB)
sigtrampnog: sigtrampnog:
// Signal arrived on a non-Go thread. If this is SIGPROF, get a // Signal arrived on a non-Go thread. If this is SIGPROF, get a
@ -486,7 +488,8 @@ sigtrampnog:
// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/unix/sysv/linux/x86_64/sigaction.c // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/unix/sysv/linux/x86_64/sigaction.c
// The code that cares about the precise instructions used is: // The code that cares about the precise instructions used is:
// https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup // https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup
TEXT runtime·sigreturn(SB),NOSPLIT,$0 // Defined as ABIInternal since it does not use the stack-based Go ABI.
TEXT runtime·sigreturn<ABIInternal>(SB),NOSPLIT,$0
MOVQ $SYS_rt_sigreturn, AX MOVQ $SYS_rt_sigreturn, AX
SYSCALL SYSCALL
INT $3 // not reached INT $3 // not reached