From 10ac80de7732bd866abce03d18f3d512ea33ba14 Mon Sep 17 00:00:00 2001 From: Ch1n-ch1nless Date: Thu, 21 Aug 2025 17:41:13 +0300 Subject: [PATCH] cmd/compile: introduce CCMP generation Introduce new aux type "ARM64ConditionalParams", which contains condition code, NZCV flags and constant with indicator of using it for CCMP instructions Updates #71268 Change-Id: I322a6cb7077c9a2c4415893c5eb7ff7692d5a2de Reviewed-on: https://go-review.googlesource.com/c/go/+/698037 Reviewed-by: Mark Freeman Reviewed-by: Keith Randall Reviewed-by: Keith Randall LUCI-TryBot-Result: Go LUCI Auto-Submit: Keith Randall --- src/cmd/compile/internal/arm64/ssa.go | 21 ++++ src/cmd/compile/internal/ssa/_gen/ARM64Ops.go | 18 ++++ src/cmd/compile/internal/ssa/_gen/rulegen.go | 4 +- src/cmd/compile/internal/ssa/check.go | 2 +- src/cmd/compile/internal/ssa/op.go | 19 +++- src/cmd/compile/internal/ssa/opGen.go | 100 ++++++++++++++++++ src/cmd/compile/internal/ssa/rewrite.go | 62 +++++++++++ src/cmd/compile/internal/ssa/value.go | 16 +++ 8 files changed, 235 insertions(+), 7 deletions(-) diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go index 402ec493c2..7bc0e536e9 100644 --- a/src/cmd/compile/internal/arm64/ssa.go +++ b/src/cmd/compile/internal/arm64/ssa.go @@ -1050,6 +1050,27 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { p.From.Offset = int64(condCode) p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() + case ssa.OpARM64CCMP, + ssa.OpARM64CCMN, + ssa.OpARM64CCMPconst, + ssa.OpARM64CCMNconst, + ssa.OpARM64CCMPW, + ssa.OpARM64CCMNW, + ssa.OpARM64CCMPWconst, + ssa.OpARM64CCMNWconst: + p := s.Prog(v.Op.Asm()) + p.Reg = v.Args[0].Reg() + params := v.AuxArm64ConditionalParams() + p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset + p.From.Offset = int64(condBits[params.Cond()]) + constValue, ok := params.ConstValue() + if ok { + p.AddRestSourceConst(constValue) + } else { + p.AddRestSourceReg(v.Args[1].Reg()) + } + p.To.Type = obj.TYPE_CONST + p.To.Offset = params.Nzcv() case ssa.OpARM64LoweredZero: ptrReg := v.Args[0].Reg() n := v.AuxInt diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go index 51aa37886b..43072ae913 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go @@ -156,12 +156,14 @@ func init() { gp11 = regInfo{inputs: []regMask{gpg}, outputs: []regMask{gp}} gp11sp = regInfo{inputs: []regMask{gpspg}, outputs: []regMask{gp}} gp1flags = regInfo{inputs: []regMask{gpg}} + gp1flagsflags = regInfo{inputs: []regMask{gpg}} gp1flags1 = regInfo{inputs: []regMask{gpg}, outputs: []regMask{gp}} gp11flags = regInfo{inputs: []regMask{gpg}, outputs: []regMask{gp, 0}} gp21 = regInfo{inputs: []regMask{gpg, gpg}, outputs: []regMask{gp}} gp21nog = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp}} gp21flags = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp, 0}} gp2flags = regInfo{inputs: []regMask{gpg, gpg}} + gp2flagsflags = regInfo{inputs: []regMask{gpg, gpg}} gp2flags1 = regInfo{inputs: []regMask{gp, gp}, outputs: []regMask{gp}} gp2flags1flags = regInfo{inputs: []regMask{gp, gp, 0}, outputs: []regMask{gp, 0}} gp2load = regInfo{inputs: []regMask{gpspsbg, gpg}, outputs: []regMask{gp}} @@ -508,6 +510,22 @@ func init() { {name: "CSNEG", argLength: 3, reg: gp2flags1, asm: "CSNEG", aux: "CCop"}, // auxint(flags) ? arg0 : -arg1 {name: "CSETM", argLength: 1, reg: readflags, asm: "CSETM", aux: "CCop"}, // auxint(flags) ? -1 : 0 + // conditional comparison instructions; auxint is + // combination of Cond, Nzcv and optional ConstValue + // Behavior: + // If the condition 'Cond' evaluates to true against current flags, + // flags are set to the result of the comparison operation. + // Otherwise, flags are set to the fallback value 'Nzcv'. + {name: "CCMP", argLength: 3, reg: gp2flagsflags, asm: "CCMP", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CMP arg0 arg1 else flags = Nzcv + {name: "CCMN", argLength: 3, reg: gp2flagsflags, asm: "CCMN", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CMN arg0 arg1 else flags = Nzcv + {name: "CCMPconst", argLength: 2, reg: gp1flagsflags, asm: "CCMP", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CMPconst [ConstValue] arg0 else flags = Nzcv + {name: "CCMNconst", argLength: 2, reg: gp1flagsflags, asm: "CCMN", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CMNconst [ConstValue] arg0 else flags = Nzcv + + {name: "CCMPW", argLength: 3, reg: gp2flagsflags, asm: "CCMPW", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CMPW arg0 arg1 else flags = Nzcv + {name: "CCMNW", argLength: 3, reg: gp2flagsflags, asm: "CCMNW", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CMNW arg0 arg1 else flags = Nzcv + {name: "CCMPWconst", argLength: 2, reg: gp1flagsflags, asm: "CCMPW", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CCMPWconst [ConstValue] arg0 else flags = Nzcv + {name: "CCMNWconst", argLength: 2, reg: gp1flagsflags, asm: "CCMNW", aux: "ARM64ConditionalParams", typ: "Flag"}, // If Cond then flags = CCMNWconst [ConstValue] arg0 else flags = Nzcv + // function calls {name: "CALLstatic", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true}, // call static function aux.(*obj.LSym). last arg=mem, auxint=argsize, returns mem {name: "CALLtail", argLength: -1, reg: regInfo{clobbers: callerSave}, aux: "CallOff", clobberFlags: true, call: true, tailCall: true}, // tail call static function aux.(*obj.LSym). last arg=mem, auxint=argsize, returns mem diff --git a/src/cmd/compile/internal/ssa/_gen/rulegen.go b/src/cmd/compile/internal/ssa/_gen/rulegen.go index d21647ad6d..f818b46511 100644 --- a/src/cmd/compile/internal/ssa/_gen/rulegen.go +++ b/src/cmd/compile/internal/ssa/_gen/rulegen.go @@ -1439,7 +1439,7 @@ func opHasAuxInt(op opData) bool { switch op.aux { case "Bool", "Int8", "Int16", "Int32", "Int64", "Int128", "UInt8", "Float32", "Float64", "SymOff", "CallOff", "SymValAndOff", "TypSize", "ARM64BitField", "FlagConstant", "CCop", - "PanicBoundsC", "PanicBoundsCC": + "PanicBoundsC", "PanicBoundsCC", "ARM64ConditionalParams": return true } return false @@ -1847,6 +1847,8 @@ func (op opData) auxIntType() string { return "flagConstant" case "ARM64BitField": return "arm64BitField" + case "ARM64ConditionalParams": + return "arm64ConditionalParams" case "PanicBoundsC", "PanicBoundsCC": return "int64" default: diff --git a/src/cmd/compile/internal/ssa/check.go b/src/cmd/compile/internal/ssa/check.go index f33c9bc87b..ef06bd9171 100644 --- a/src/cmd/compile/internal/ssa/check.go +++ b/src/cmd/compile/internal/ssa/check.go @@ -145,7 +145,7 @@ func checkFunc(f *Func) { f.Fatalf("bad int32 AuxInt value for %v", v) } canHaveAuxInt = true - case auxInt64, auxARM64BitField: + case auxInt64, auxARM64BitField, auxARM64ConditionalParams: canHaveAuxInt = true case auxInt128: // AuxInt must be zero, so leave canHaveAuxInt set to false. diff --git a/src/cmd/compile/internal/ssa/op.go b/src/cmd/compile/internal/ssa/op.go index e8bd5d9acf..b279cf06bc 100644 --- a/src/cmd/compile/internal/ssa/op.go +++ b/src/cmd/compile/internal/ssa/op.go @@ -375,11 +375,12 @@ const ( auxPanicBoundsCC // two constants for a bounds failure // architecture specific aux types - auxARM64BitField // aux is an arm64 bitfield lsb and width packed into auxInt - auxS390XRotateParams // aux is a s390x rotate parameters object encoding start bit, end bit and rotate amount - auxS390XCCMask // aux is a s390x 4-bit condition code mask - auxS390XCCMaskInt8 // aux is a s390x 4-bit condition code mask, auxInt is an int8 immediate - auxS390XCCMaskUint8 // aux is a s390x 4-bit condition code mask, auxInt is a uint8 immediate + auxARM64BitField // aux is an arm64 bitfield lsb and width packed into auxInt + auxARM64ConditionalParams // aux is a structure, which contains condition, NZCV flags and constant with indicator of using it + auxS390XRotateParams // aux is a s390x rotate parameters object encoding start bit, end bit and rotate amount + auxS390XCCMask // aux is a s390x 4-bit condition code mask + auxS390XCCMaskInt8 // aux is a s390x 4-bit condition code mask, auxInt is an int8 immediate + auxS390XCCMaskUint8 // aux is a s390x 4-bit condition code mask, auxInt is a uint8 immediate ) // A SymEffect describes the effect that an SSA Value has on the variable @@ -534,3 +535,11 @@ func (b BoundsKind) Code() (rtabi.BoundsErrorCode, bool) { // width+lsb<64 for 64-bit variant, width+lsb<32 for 32-bit variant. // the meaning of width and lsb are instruction-dependent. type arm64BitField int16 + +// arm64ConditionalParams is the GO type of ARM64ConditionalParams auxInt. +type arm64ConditionalParams struct { + cond Op // Condition code to evaluate + nzcv uint8 // Fallback NZCV flags value when condition is false + constValue uint8 // Immediate value for constant comparisons + ind bool // Constant comparison indicator +} diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index d9cccb27ba..4742525805 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -1693,6 +1693,14 @@ const ( OpARM64CSINV OpARM64CSNEG OpARM64CSETM + OpARM64CCMP + OpARM64CCMN + OpARM64CCMPconst + OpARM64CCMNconst + OpARM64CCMPW + OpARM64CCMNW + OpARM64CCMPWconst + OpARM64CCMNWconst OpARM64CALLstatic OpARM64CALLtail OpARM64CALLclosure @@ -22829,6 +22837,98 @@ var opcodeTable = [...]opInfo{ }, }, }, + { + name: "CCMP", + auxType: auxARM64ConditionalParams, + argLen: 3, + asm: arm64.ACCMP, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + {1, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMN", + auxType: auxARM64ConditionalParams, + argLen: 3, + asm: arm64.ACCMN, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + {1, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMPconst", + auxType: auxARM64ConditionalParams, + argLen: 2, + asm: arm64.ACCMP, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMNconst", + auxType: auxARM64ConditionalParams, + argLen: 2, + asm: arm64.ACCMN, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMPW", + auxType: auxARM64ConditionalParams, + argLen: 3, + asm: arm64.ACCMPW, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + {1, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMNW", + auxType: auxARM64ConditionalParams, + argLen: 3, + asm: arm64.ACCMNW, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + {1, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMPWconst", + auxType: auxARM64ConditionalParams, + argLen: 2, + asm: arm64.ACCMPW, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, + { + name: "CCMNWconst", + auxType: auxARM64ConditionalParams, + argLen: 2, + asm: arm64.ACCMNW, + reg: regInfo{ + inputs: []inputInfo{ + {0, 402653183}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 + }, + }, + }, { name: "CALLstatic", auxType: auxCallOff, diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index 576f25a497..6fb2689d18 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -669,6 +669,17 @@ func auxIntToValAndOff(i int64) ValAndOff { func auxIntToArm64BitField(i int64) arm64BitField { return arm64BitField(i) } +func auxIntToArm64ConditionalParams(i int64) arm64ConditionalParams { + var params arm64ConditionalParams + params.cond = Op(i & 0xffff) + i >>= 16 + params.nzcv = uint8(i & 0x0f) + i >>= 4 + params.constValue = uint8(i & 0x1f) + i >>= 5 + params.ind = i == 1 + return params +} func auxIntToFlagConstant(x int64) flagConstant { return flagConstant(x) } @@ -710,6 +721,20 @@ func valAndOffToAuxInt(v ValAndOff) int64 { func arm64BitFieldToAuxInt(v arm64BitField) int64 { return int64(v) } +func arm64ConditionalParamsToAuxInt(v arm64ConditionalParams) int64 { + if v.cond&^0xffff != 0 { + panic("condition value exceeds 16 bits") + } + + var i int64 + if v.ind { + i = 1 << 25 + } + i |= int64(v.constValue) << 20 + i |= int64(v.nzcv) << 16 + i |= int64(v.cond) + return i +} func flagConstantToAuxInt(x flagConstant) int64 { return int64(x) } @@ -1899,6 +1924,43 @@ func arm64BFWidth(mask, rshift int64) int64 { return nto(shiftedMask) } +// encodes condition code and NZCV flags into auxint. +func arm64ConditionalParamsAuxInt(cond Op, nzcv uint8) arm64ConditionalParams { + if cond < OpARM64Equal || cond > OpARM64GreaterEqualU { + panic("Wrong conditional operation") + } + if nzcv&0x0f != nzcv { + panic("Wrong value of NZCV flag") + } + return arm64ConditionalParams{cond, nzcv, 0, false} +} + +// encodes condition code, NZCV flags and constant value into auxint. +func arm64ConditionalParamsAuxIntWithValue(cond Op, nzcv uint8, value uint8) arm64ConditionalParams { + if value&0x1f != value { + panic("Wrong value of constant") + } + params := arm64ConditionalParamsAuxInt(cond, nzcv) + params.constValue = value + params.ind = true + return params +} + +// extracts condition code from auxint. +func (condParams arm64ConditionalParams) Cond() Op { + return condParams.cond +} + +// extracts NZCV flags from auxint. +func (condParams arm64ConditionalParams) Nzcv() int64 { + return int64(condParams.nzcv) +} + +// extracts constant value from auxint if present. +func (condParams arm64ConditionalParams) ConstValue() (int64, bool) { + return int64(condParams.constValue), condParams.ind +} + // registerizable reports whether t is a primitive type that fits in // a register. It assumes float64 values will always fit into registers // even if that isn't strictly true. diff --git a/src/cmd/compile/internal/ssa/value.go b/src/cmd/compile/internal/ssa/value.go index e80b712ddb..55ab23ce9a 100644 --- a/src/cmd/compile/internal/ssa/value.go +++ b/src/cmd/compile/internal/ssa/value.go @@ -144,6 +144,13 @@ func (v *Value) AuxArm64BitField() arm64BitField { return arm64BitField(v.AuxInt) } +func (v *Value) AuxArm64ConditionalParams() arm64ConditionalParams { + if opcodeTable[v.Op].auxType != auxARM64ConditionalParams { + v.Fatalf("op %s doesn't have a ARM64ConditionalParams aux field", v.Op) + } + return auxIntToArm64ConditionalParams(v.AuxInt) +} + // long form print. v# = opcode [aux] args [: reg] (names) func (v *Value) LongString() string { if v == nil { @@ -203,6 +210,15 @@ func (v *Value) auxString() string { lsb := v.AuxArm64BitField().lsb() width := v.AuxArm64BitField().width() return fmt.Sprintf(" [lsb=%d,width=%d]", lsb, width) + case auxARM64ConditionalParams: + params := v.AuxArm64ConditionalParams() + cond := params.Cond() + nzcv := params.Nzcv() + imm, ok := params.ConstValue() + if ok { + return fmt.Sprintf(" [cond=%s,nzcv=%d,imm=%d]", cond, nzcv, imm) + } + return fmt.Sprintf(" [cond=%s,nzcv=%d]", cond, nzcv) case auxFloat32, auxFloat64: return fmt.Sprintf(" [%g]", v.AuxFloat()) case auxString: