cmd/compile: move riscv64 over to new bounds check strategy

Change-Id: Idd9eaf051aa57f7fef7049c12085926030c35d70
Reviewed-on: https://go-review.googlesource.com/c/go/+/682401
Reviewed-by: Mark Freeman <mark@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Joel Sing <joel@sing.id.au>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
This commit is contained in:
Keith Randall 2025-06-18 16:35:48 -07:00
parent d7bd7773eb
commit 95693816a5
6 changed files with 238 additions and 159 deletions

View file

@ -14,6 +14,7 @@ import (
"cmd/compile/internal/types"
"cmd/internal/obj"
"cmd/internal/obj/riscv"
"internal/abi"
)
// ssaRegToReg maps ssa register numbers to obj register numbers.
@ -508,12 +509,91 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.To.Name = obj.NAME_EXTERN
// AuxInt encodes how many buffer entries we need.
p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
case ssa.OpRISCV64LoweredPanicBoundsA, ssa.OpRISCV64LoweredPanicBoundsB, ssa.OpRISCV64LoweredPanicBoundsC:
p := s.Prog(obj.ACALL)
case ssa.OpRISCV64LoweredPanicBoundsRR, ssa.OpRISCV64LoweredPanicBoundsRC, ssa.OpRISCV64LoweredPanicBoundsCR, ssa.OpRISCV64LoweredPanicBoundsCC:
// Compute the constant we put in the PCData entry for this call.
code, signed := ssa.BoundsKind(v.AuxInt).Code()
xIsReg := false
yIsReg := false
xVal := 0
yVal := 0
switch v.Op {
case ssa.OpRISCV64LoweredPanicBoundsRR:
xIsReg = true
xVal = int(v.Args[0].Reg() - riscv.REG_X5)
yIsReg = true
yVal = int(v.Args[1].Reg() - riscv.REG_X5)
case ssa.OpRISCV64LoweredPanicBoundsRC:
xIsReg = true
xVal = int(v.Args[0].Reg() - riscv.REG_X5)
c := v.Aux.(ssa.PanicBoundsC).C
if c >= 0 && c <= abi.BoundsMaxConst {
yVal = int(c)
} else {
// Move constant to a register
yIsReg = true
if yVal == xVal {
yVal = 1
}
p := s.Prog(riscv.AMOV)
p.From.Type = obj.TYPE_CONST
p.From.Offset = c
p.To.Type = obj.TYPE_REG
p.To.Reg = riscv.REG_X5 + int16(yVal)
}
case ssa.OpRISCV64LoweredPanicBoundsCR:
yIsReg = true
yVal := int(v.Args[0].Reg() - riscv.REG_X5)
c := v.Aux.(ssa.PanicBoundsC).C
if c >= 0 && c <= abi.BoundsMaxConst {
xVal = int(c)
} else {
// Move constant to a register
if xVal == yVal {
xVal = 1
}
p := s.Prog(riscv.AMOV)
p.From.Type = obj.TYPE_CONST
p.From.Offset = c
p.To.Type = obj.TYPE_REG
p.To.Reg = riscv.REG_X5 + int16(xVal)
}
case ssa.OpRISCV64LoweredPanicBoundsCC:
c := v.Aux.(ssa.PanicBoundsCC).Cx
if c >= 0 && c <= abi.BoundsMaxConst {
xVal = int(c)
} else {
// Move constant to a register
xIsReg = true
p := s.Prog(riscv.AMOV)
p.From.Type = obj.TYPE_CONST
p.From.Offset = c
p.To.Type = obj.TYPE_REG
p.To.Reg = riscv.REG_X5 + int16(xVal)
}
c = v.Aux.(ssa.PanicBoundsCC).Cy
if c >= 0 && c <= abi.BoundsMaxConst {
yVal = int(c)
} else {
// Move constant to a register
yIsReg = true
yVal = 1
p := s.Prog(riscv.AMOV)
p.From.Type = obj.TYPE_CONST
p.From.Offset = c
p.To.Type = obj.TYPE_REG
p.To.Reg = riscv.REG_X5 + int16(yVal)
}
}
c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
p := s.Prog(obj.APCDATA)
p.From.SetConst(abi.PCDATA_PanicBounds)
p.To.SetConst(int64(c))
p = s.Prog(obj.ACALL)
p.To.Type = obj.TYPE_MEM
p.To.Name = obj.NAME_EXTERN
p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt]
s.UseArgs(16) // space used in callee args area by assembly stubs
p.To.Sym = ir.Syms.PanicBounds
case ssa.OpRISCV64LoweredAtomicLoad8:
s.Prog(riscv.AFENCE)

View file

@ -407,9 +407,11 @@
// Publication barrier as intrinsic
(PubBarrier ...) => (LoweredPubBarrier ...)
(PanicBounds [kind] x y mem) && boundsABI(kind) == 0 => (LoweredPanicBoundsA [kind] x y mem)
(PanicBounds [kind] x y mem) && boundsABI(kind) == 1 => (LoweredPanicBoundsB [kind] x y mem)
(PanicBounds [kind] x y mem) && boundsABI(kind) == 2 => (LoweredPanicBoundsC [kind] x y mem)
(PanicBounds ...) => (LoweredPanicBoundsRR ...)
(LoweredPanicBoundsRR [kind] x (MOVDconst [c]) mem) => (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:c}} mem)
(LoweredPanicBoundsRR [kind] (MOVDconst [c]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:c}} y mem)
(LoweredPanicBoundsRC [kind] {p} (MOVDconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:c, Cy:p.C}} mem)
(LoweredPanicBoundsCR [kind] {p} (MOVDconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:p.C, Cy:c}} mem)
// Small moves
(Move [0] _ _ mem) => mem

View file

@ -49,7 +49,7 @@ func riscv64RegName(r int) string {
func init() {
var regNamesRISCV64 []string
var gpMask, fpMask, gpgMask, gpspMask, gpspsbMask, gpspsbgMask regMask
var gpMask, fpMask, gpgMask, gpspMask, gpspsbMask, gpspsbgMask, first16Mask regMask
regNamed := make(map[string]regMask)
// Build the list of register names, creating an appropriately indexed
@ -93,6 +93,9 @@ func init() {
gpspMask |= mask
gpspsbMask |= mask
gpspsbgMask |= mask
if r >= 5 && r < 5+16 {
first16Mask |= mask
}
}
}
@ -429,12 +432,15 @@ func init() {
// Do data barrier. arg0=memorys
{name: "LoweredPubBarrier", argLength: 1, asm: "FENCE", hasSideEffects: true},
// There are three of these functions so that they can have three different register inputs.
// When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the
// default registers to match so we don't need to copy registers around unnecessarily.
{name: "LoweredPanicBoundsA", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{regNamed["X7"], regNamed["X28"]}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go).
{name: "LoweredPanicBoundsB", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{regNamed["X6"], regNamed["X7"]}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go).
{name: "LoweredPanicBoundsC", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{regNamed["X5"], regNamed["X6"]}}, typ: "Mem", call: true}, // arg0=idx, arg1=len, arg2=mem, returns memory. AuxInt contains report code (see PanicBounds in genericOps.go).
// LoweredPanicBoundsRR takes x and y, two values that caused a bounds check to fail.
// the RC and CR versions are used when one of the arguments is a constant. CC is used
// when both are constant (normally both 0, as prove derives the fact that a [0] bounds
// failure means the length must have also been 0).
// AuxInt contains a report code (see PanicBounds in genericOps.go).
{name: "LoweredPanicBoundsRR", argLength: 3, aux: "Int64", reg: regInfo{inputs: []regMask{first16Mask, first16Mask}}, typ: "Mem", call: true}, // arg0=x, arg1=y, arg2=mem, returns memory.
{name: "LoweredPanicBoundsRC", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{first16Mask}}, typ: "Mem", call: true}, // arg0=x, arg1=mem, returns memory.
{name: "LoweredPanicBoundsCR", argLength: 2, aux: "PanicBoundsC", reg: regInfo{inputs: []regMask{first16Mask}}, typ: "Mem", call: true}, // arg0=y, arg1=mem, returns memory.
{name: "LoweredPanicBoundsCC", argLength: 1, aux: "PanicBoundsCC", reg: regInfo{}, typ: "Mem", call: true}, // arg0=mem, returns memory.
// F extension.
{name: "FADDS", argLength: 2, reg: fp21, asm: "FADDS", commutative: true, typ: "Float32"}, // arg0 + arg1

View file

@ -2585,9 +2585,10 @@ const (
OpRISCV64LoweredGetCallerPC
OpRISCV64LoweredWB
OpRISCV64LoweredPubBarrier
OpRISCV64LoweredPanicBoundsA
OpRISCV64LoweredPanicBoundsB
OpRISCV64LoweredPanicBoundsC
OpRISCV64LoweredPanicBoundsRR
OpRISCV64LoweredPanicBoundsRC
OpRISCV64LoweredPanicBoundsCR
OpRISCV64LoweredPanicBoundsCC
OpRISCV64FADDS
OpRISCV64FSUBS
OpRISCV64FMULS
@ -34782,41 +34783,46 @@ var opcodeTable = [...]opInfo{
reg: regInfo{},
},
{
name: "LoweredPanicBoundsA",
name: "LoweredPanicBoundsRR",
auxType: auxInt64,
argLen: 3,
call: true,
reg: regInfo{
inputs: []inputInfo{
{0, 64}, // X7
{1, 134217728}, // X28
{0, 1048560}, // X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20
{1, 1048560}, // X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20
},
},
},
{
name: "LoweredPanicBoundsB",
auxType: auxInt64,
argLen: 3,
name: "LoweredPanicBoundsRC",
auxType: auxPanicBoundsC,
argLen: 2,
call: true,
reg: regInfo{
inputs: []inputInfo{
{0, 32}, // X6
{1, 64}, // X7
{0, 1048560}, // X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20
},
},
},
{
name: "LoweredPanicBoundsC",
auxType: auxInt64,
argLen: 3,
name: "LoweredPanicBoundsCR",
auxType: auxPanicBoundsC,
argLen: 2,
call: true,
reg: regInfo{
inputs: []inputInfo{
{0, 16}, // X5
{1, 32}, // X6
{0, 1048560}, // X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20
},
},
},
{
name: "LoweredPanicBoundsCC",
auxType: auxPanicBoundsCC,
argLen: 1,
call: true,
reg: regInfo{},
},
{
name: "FADDS",
argLen: 2,

View file

@ -486,7 +486,8 @@ func rewriteValueRISCV64(v *Value) bool {
v.Op = OpRISCV64OR
return true
case OpPanicBounds:
return rewriteValueRISCV64_OpPanicBounds(v)
v.Op = OpRISCV64LoweredPanicBoundsRR
return true
case OpPopCount16:
return rewriteValueRISCV64_OpPopCount16(v)
case OpPopCount32:
@ -532,6 +533,12 @@ func rewriteValueRISCV64(v *Value) bool {
return rewriteValueRISCV64_OpRISCV64FSUBD(v)
case OpRISCV64FSUBS:
return rewriteValueRISCV64_OpRISCV64FSUBS(v)
case OpRISCV64LoweredPanicBoundsCR:
return rewriteValueRISCV64_OpRISCV64LoweredPanicBoundsCR(v)
case OpRISCV64LoweredPanicBoundsRC:
return rewriteValueRISCV64_OpRISCV64LoweredPanicBoundsRC(v)
case OpRISCV64LoweredPanicBoundsRR:
return rewriteValueRISCV64_OpRISCV64LoweredPanicBoundsRR(v)
case OpRISCV64MOVBUload:
return rewriteValueRISCV64_OpRISCV64MOVBUload(v)
case OpRISCV64MOVBUreg:
@ -3416,60 +3423,6 @@ func rewriteValueRISCV64_OpOffPtr(v *Value) bool {
return true
}
}
func rewriteValueRISCV64_OpPanicBounds(v *Value) bool {
v_2 := v.Args[2]
v_1 := v.Args[1]
v_0 := v.Args[0]
// match: (PanicBounds [kind] x y mem)
// cond: boundsABI(kind) == 0
// result: (LoweredPanicBoundsA [kind] x y mem)
for {
kind := auxIntToInt64(v.AuxInt)
x := v_0
y := v_1
mem := v_2
if !(boundsABI(kind) == 0) {
break
}
v.reset(OpRISCV64LoweredPanicBoundsA)
v.AuxInt = int64ToAuxInt(kind)
v.AddArg3(x, y, mem)
return true
}
// match: (PanicBounds [kind] x y mem)
// cond: boundsABI(kind) == 1
// result: (LoweredPanicBoundsB [kind] x y mem)
for {
kind := auxIntToInt64(v.AuxInt)
x := v_0
y := v_1
mem := v_2
if !(boundsABI(kind) == 1) {
break
}
v.reset(OpRISCV64LoweredPanicBoundsB)
v.AuxInt = int64ToAuxInt(kind)
v.AddArg3(x, y, mem)
return true
}
// match: (PanicBounds [kind] x y mem)
// cond: boundsABI(kind) == 2
// result: (LoweredPanicBoundsC [kind] x y mem)
for {
kind := auxIntToInt64(v.AuxInt)
x := v_0
y := v_1
mem := v_2
if !(boundsABI(kind) == 2) {
break
}
v.reset(OpRISCV64LoweredPanicBoundsC)
v.AuxInt = int64ToAuxInt(kind)
v.AddArg3(x, y, mem)
return true
}
return false
}
func rewriteValueRISCV64_OpPopCount16(v *Value) bool {
v_0 := v.Args[0]
b := v.Block
@ -4239,6 +4192,86 @@ func rewriteValueRISCV64_OpRISCV64FSUBS(v *Value) bool {
}
return false
}
func rewriteValueRISCV64_OpRISCV64LoweredPanicBoundsCR(v *Value) bool {
v_1 := v.Args[1]
v_0 := v.Args[0]
// match: (LoweredPanicBoundsCR [kind] {p} (MOVDconst [c]) mem)
// result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:p.C, Cy:c}} mem)
for {
kind := auxIntToInt64(v.AuxInt)
p := auxToPanicBoundsC(v.Aux)
if v_0.Op != OpRISCV64MOVDconst {
break
}
c := auxIntToInt64(v_0.AuxInt)
mem := v_1
v.reset(OpRISCV64LoweredPanicBoundsCC)
v.AuxInt = int64ToAuxInt(kind)
v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: p.C, Cy: c})
v.AddArg(mem)
return true
}
return false
}
func rewriteValueRISCV64_OpRISCV64LoweredPanicBoundsRC(v *Value) bool {
v_1 := v.Args[1]
v_0 := v.Args[0]
// match: (LoweredPanicBoundsRC [kind] {p} (MOVDconst [c]) mem)
// result: (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:c, Cy:p.C}} mem)
for {
kind := auxIntToInt64(v.AuxInt)
p := auxToPanicBoundsC(v.Aux)
if v_0.Op != OpRISCV64MOVDconst {
break
}
c := auxIntToInt64(v_0.AuxInt)
mem := v_1
v.reset(OpRISCV64LoweredPanicBoundsCC)
v.AuxInt = int64ToAuxInt(kind)
v.Aux = panicBoundsCCToAux(PanicBoundsCC{Cx: c, Cy: p.C})
v.AddArg(mem)
return true
}
return false
}
func rewriteValueRISCV64_OpRISCV64LoweredPanicBoundsRR(v *Value) bool {
v_2 := v.Args[2]
v_1 := v.Args[1]
v_0 := v.Args[0]
// match: (LoweredPanicBoundsRR [kind] x (MOVDconst [c]) mem)
// result: (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:c}} mem)
for {
kind := auxIntToInt64(v.AuxInt)
x := v_0
if v_1.Op != OpRISCV64MOVDconst {
break
}
c := auxIntToInt64(v_1.AuxInt)
mem := v_2
v.reset(OpRISCV64LoweredPanicBoundsRC)
v.AuxInt = int64ToAuxInt(kind)
v.Aux = panicBoundsCToAux(PanicBoundsC{C: c})
v.AddArg2(x, mem)
return true
}
// match: (LoweredPanicBoundsRR [kind] (MOVDconst [c]) y mem)
// result: (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:c}} y mem)
for {
kind := auxIntToInt64(v.AuxInt)
if v_0.Op != OpRISCV64MOVDconst {
break
}
c := auxIntToInt64(v_0.AuxInt)
y := v_1
mem := v_2
v.reset(OpRISCV64LoweredPanicBoundsCR)
v.AuxInt = int64ToAuxInt(kind)
v.Aux = panicBoundsCToAux(PanicBoundsC{C: c})
v.AddArg2(y, mem)
return true
}
return false
}
func rewriteValueRISCV64_OpRISCV64MOVBUload(v *Value) bool {
v_1 := v.Args[1]
v_0 := v.Args[0]

View file

@ -884,80 +884,32 @@ TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
MOV $64, X24
JMP gcWriteBarrier<>(SB)
// Note: these functions use a special calling convention to save generated code space.
// Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those
// arguments are allocated in the caller's stack frame.
// These stubs write the args into that stack space and then tail call to the
// corresponding runtime handler.
// The tail call makes these stubs disappear in backtraces.
TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
MOV T0, X10
MOV T1, X11
JMP runtime·goPanicIndex<ABIInternal>(SB)
TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T0, X10
MOV T1, X11
JMP runtime·goPanicIndexU<ABIInternal>(SB)
TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
MOV T1, X10
MOV T2, X11
JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T1, X10
MOV T2, X11
JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
MOV T1, X10
MOV T2, X11
JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T1, X10
MOV T2, X11
JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
MOV T0, X10
MOV T1, X11
JMP runtime·goPanicSliceB<ABIInternal>(SB)
TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T0, X10
MOV T1, X11
JMP runtime·goPanicSliceBU<ABIInternal>(SB)
TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
MOV T2, X10
MOV T3, X11
JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T2, X10
MOV T3, X11
JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
MOV T2, X10
MOV T3, X11
JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T2, X10
MOV T3, X11
JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
MOV T1, X10
MOV T2, X11
JMP runtime·goPanicSlice3B<ABIInternal>(SB)
TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T1, X10
MOV T2, X11
JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
MOV T0, X10
MOV T1, X11
JMP runtime·goPanicSlice3C<ABIInternal>(SB)
TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
MOV T0, X10
MOV T1, X11
JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
MOV T2, X10
MOV T3, X11
JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
NO_LOCAL_POINTERS
// Save all 16 int registers that could have an index in them.
// They may be pointers, but if they are they are dead.
// Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
MOV X5, 24(X2)
MOV X6, 32(X2)
MOV X7, 40(X2)
MOV X8, 48(X2)
MOV X9, 56(X2)
MOV X10, 64(X2)
MOV X11, 72(X2)
MOV X12, 80(X2)
MOV X13, 88(X2)
MOV X14, 96(X2)
MOV X15, 104(X2)
MOV X16, 112(X2)
MOV X17, 120(X2)
MOV X18, 128(X2)
MOV X19, 136(X2)
MOV X20, 144(X2)
MOV X1, X10 // PC immediately after call to panicBounds
ADD $24, X2, X11 // pointer to save area
CALL runtime·panicBounds64<ABIInternal>(SB)
RET
DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
GLOBL runtime·mainPC(SB),RODATA,$8