2021-11-24 17:31:18 +08:00
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import "strings"
// Notes:
// - Integer types live in the low portion of registers. Upper portions are junk.
// - Boolean types use the low-order byte of a register. 0=false, 1=true.
// Upper bytes are junk.
// - *const instructions may use a constant larger than the instruction can encode.
// In this case the assembler expands to multiple instructions and uses tmp
// register (R23).
// Suffixes encode the bit width of various instructions.
// V (vlong) = 64 bit
// WU (word) = 32 bit unsigned
// W (word) = 32 bit
// H (half word) = 16 bit
// HU = 16 bit unsigned
// B (byte) = 8 bit
// BU = 8 bit unsigned
// F (float) = 32 bit float
// D (double) = 64 bit float
// Note: registers not used in regalloc are not included in this list,
// so that regmask stays within int64
// Be careful when hand coding regmasks.
var regNamesLOONG64 = [ ] string {
"R0" , // constant 0
"R1" ,
"SP" , // aka R3
"R4" ,
"R5" ,
"R6" ,
"R7" ,
"R8" ,
"R9" ,
"R10" ,
"R11" ,
"R12" ,
"R13" ,
"R14" ,
"R15" ,
"R16" ,
"R17" ,
"R18" ,
"R19" ,
"R20" ,
"R21" ,
"g" , // aka R22
"R23" ,
"R24" ,
"R25" ,
"R26" ,
"R27" ,
"R28" ,
"R29" ,
// R30 is REGTMP not used in regalloc
"R31" ,
"F0" ,
"F1" ,
"F2" ,
"F3" ,
"F4" ,
"F5" ,
"F6" ,
"F7" ,
"F8" ,
"F9" ,
"F10" ,
"F11" ,
"F12" ,
"F13" ,
"F14" ,
"F15" ,
"F16" ,
"F17" ,
"F18" ,
"F19" ,
"F20" ,
"F21" ,
"F22" ,
"F23" ,
"F24" ,
"F25" ,
"F26" ,
"F27" ,
"F28" ,
"F29" ,
"F30" ,
"F31" ,
// If you add registers, update asyncPreempt in runtime.
// pseudo-registers
"SB" ,
}
func init ( ) {
// Make map from reg names to reg integers.
if len ( regNamesLOONG64 ) > 64 {
panic ( "too many registers" )
}
num := map [ string ] int { }
for i , name := range regNamesLOONG64 {
num [ name ] = i
}
buildReg := func ( s string ) regMask {
m := regMask ( 0 )
for _ , r := range strings . Split ( s , " " ) {
if n , ok := num [ r ] ; ok {
m |= regMask ( 1 ) << uint ( n )
continue
}
panic ( "register " + r + " not found" )
}
return m
}
// Common individual register masks
var (
2023-08-17 03:58:10 +08:00
gp = buildReg ( "R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19 R20 R21 R23 R24 R25 R26 R27 R28 R29 R31" ) // R1 is LR, R2 is thread pointer, R3 is stack pointer, R22 is g, R30 is REGTMP
2021-11-24 17:31:18 +08:00
gpg = gp | buildReg ( "g" )
gpsp = gp | buildReg ( "SP" )
gpspg = gpg | buildReg ( "SP" )
gpspsbg = gpspg | buildReg ( "SB" )
fp = buildReg ( "F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31" )
callerSave = gp | fp | buildReg ( "g" ) // runtime.setg (and anything calling it) may clobber g
2025-06-18 15:16:59 -07:00
first16 = buildReg ( "R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19" )
2021-11-24 17:31:18 +08:00
)
// Common regInfo
var (
gp01 = regInfo { inputs : nil , outputs : [ ] regMask { gp } }
gp11 = regInfo { inputs : [ ] regMask { gpg } , outputs : [ ] regMask { gp } }
gp11sp = regInfo { inputs : [ ] regMask { gpspg } , outputs : [ ] regMask { gp } }
gp21 = regInfo { inputs : [ ] regMask { gpg , gpg } , outputs : [ ] regMask { gp } }
gpload = regInfo { inputs : [ ] regMask { gpspsbg } , outputs : [ ] regMask { gp } }
2024-09-26 14:17:17 +08:00
gp2load = regInfo { inputs : [ ] regMask { gpspsbg , gpg } , outputs : [ ] regMask { gp } }
2021-11-24 17:31:18 +08:00
gpstore = regInfo { inputs : [ ] regMask { gpspsbg , gpg } }
gpstore0 = regInfo { inputs : [ ] regMask { gpspsbg } }
2024-09-26 14:17:17 +08:00
gpstore2 = regInfo { inputs : [ ] regMask { gpspsbg , gpg , gpg } }
2021-11-24 17:31:18 +08:00
gpxchg = regInfo { inputs : [ ] regMask { gpspsbg , gpg } , outputs : [ ] regMask { gp } }
gpcas = regInfo { inputs : [ ] regMask { gpspsbg , gpg , gpg } , outputs : [ ] regMask { gp } }
2025-05-12 11:01:44 +08:00
preldreg = regInfo { inputs : [ ] regMask { gpspg } }
2021-11-24 17:31:18 +08:00
fp01 = regInfo { inputs : nil , outputs : [ ] regMask { fp } }
fp11 = regInfo { inputs : [ ] regMask { fp } , outputs : [ ] regMask { fp } }
fp21 = regInfo { inputs : [ ] regMask { fp , fp } , outputs : [ ] regMask { fp } }
2024-11-05 15:30:45 +08:00
fp31 = regInfo { inputs : [ ] regMask { fp , fp , fp } , outputs : [ ] regMask { fp } }
2021-11-24 17:31:18 +08:00
fp2flags = regInfo { inputs : [ ] regMask { fp , fp } }
fpload = regInfo { inputs : [ ] regMask { gpspsbg } , outputs : [ ] regMask { fp } }
2024-09-26 14:17:17 +08:00
fp2load = regInfo { inputs : [ ] regMask { gpspsbg , gpg } , outputs : [ ] regMask { fp } }
2021-11-24 17:31:18 +08:00
fpstore = regInfo { inputs : [ ] regMask { gpspsbg , fp } }
2024-09-26 14:17:17 +08:00
fpstore2 = regInfo { inputs : [ ] regMask { gpspsbg , gpg , fp } }
2024-08-12 16:41:11 +08:00
fpgp = regInfo { inputs : [ ] regMask { fp } , outputs : [ ] regMask { gp } }
gpfp = regInfo { inputs : [ ] regMask { gp } , outputs : [ ] regMask { fp } }
2021-11-24 17:31:18 +08:00
readflags = regInfo { inputs : nil , outputs : [ ] regMask { gp } }
)
ops := [ ] opData {
2024-10-18 16:31:29 +08:00
// unary ops
{ name : "NEGV" , argLength : 1 , reg : gp11 } , // -arg0
{ name : "NEGF" , argLength : 1 , reg : fp11 , asm : "NEGF" } , // -arg0, float32
{ name : "NEGD" , argLength : 1 , reg : fp11 , asm : "NEGD" } , // -arg0, float64
{ name : "SQRTD" , argLength : 1 , reg : fp11 , asm : "SQRTD" } , // sqrt(arg0), float64
{ name : "SQRTF" , argLength : 1 , reg : fp11 , asm : "SQRTF" } , // sqrt(arg0), float32
{ name : "ABSD" , argLength : 1 , reg : fp11 , asm : "ABSD" } , // abs(arg0), float64
{ name : "CLZW" , argLength : 1 , reg : gp11 , asm : "CLZW" } , // Count leading (high order) zeroes (returns 0-32)
{ name : "CLZV" , argLength : 1 , reg : gp11 , asm : "CLZV" } , // Count leading (high order) zeroes (returns 0-64)
2024-11-01 16:09:32 +08:00
{ name : "CTZW" , argLength : 1 , reg : gp11 , asm : "CTZW" } , // Count trailing (low order) zeroes (returns 0-32)
{ name : "CTZV" , argLength : 1 , reg : gp11 , asm : "CTZV" } , // Count trailing (low order) zeroes (returns 0-64)
2024-10-18 16:31:29 +08:00
{ name : "REVB2H" , argLength : 1 , reg : gp11 , asm : "REVB2H" } , // Swap bytes: 0x11223344 -> 0x22114433 (sign extends to 64 bits)
{ name : "REVB2W" , argLength : 1 , reg : gp11 , asm : "REVB2W" } , // Swap bytes: 0x1122334455667788 -> 0x4433221188776655
{ name : "REVBV" , argLength : 1 , reg : gp11 , asm : "REVBV" } , // Swap bytes: 0x1122334455667788 -> 0x8877665544332211
{ name : "BITREV4B" , argLength : 1 , reg : gp11 , asm : "BITREV4B" } , // Reverse the bits of each byte inside a 32-bit arg[0]
{ name : "BITREVW" , argLength : 1 , reg : gp11 , asm : "BITREVW" } , // Reverse the bits in a 32-bit arg[0]
{ name : "BITREVV" , argLength : 1 , reg : gp11 , asm : "BITREVV" } , // Reverse the bits in a 64-bit arg[0]
{ name : "VPCNT64" , argLength : 1 , reg : fp11 , asm : "VPCNTV" } , // count set bits for each 64-bit unit and store the result in each 64-bit unit
{ name : "VPCNT32" , argLength : 1 , reg : fp11 , asm : "VPCNTW" } , // count set bits for each 32-bit unit and store the result in each 32-bit unit
{ name : "VPCNT16" , argLength : 1 , reg : fp11 , asm : "VPCNTH" } , // count set bits for each 16-bit unit and store the result in each 16-bit unit
2021-11-24 17:31:18 +08:00
// binary ops
{ name : "ADDV" , argLength : 2 , reg : gp21 , asm : "ADDVU" , commutative : true } , // arg0 + arg1
{ name : "ADDVconst" , argLength : 1 , reg : gp11sp , asm : "ADDVU" , aux : "Int64" } , // arg0 + auxInt. auxInt is 32-bit, also in other *const ops.
{ name : "SUBV" , argLength : 2 , reg : gp21 , asm : "SUBVU" } , // arg0 - arg1
{ name : "SUBVconst" , argLength : 1 , reg : gp11 , asm : "SUBVU" , aux : "Int64" } , // arg0 - auxInt
2023-03-02 13:33:21 +08:00
{ name : "MULV" , argLength : 2 , reg : gp21 , asm : "MULV" , commutative : true , typ : "Int64" } , // arg0 * arg1
{ name : "MULHV" , argLength : 2 , reg : gp21 , asm : "MULHV" , commutative : true , typ : "Int64" } , // (arg0 * arg1) >> 64, signed
{ name : "MULHVU" , argLength : 2 , reg : gp21 , asm : "MULHVU" , commutative : true , typ : "UInt64" } , // (arg0 * arg1) >> 64, unsigned
2023-03-02 15:55:03 +08:00
{ name : "DIVV" , argLength : 2 , reg : gp21 , asm : "DIVV" , typ : "Int64" } , // arg0 / arg1, signed
{ name : "DIVVU" , argLength : 2 , reg : gp21 , asm : "DIVVU" , typ : "UInt64" } , // arg0 / arg1, unsigned
{ name : "REMV" , argLength : 2 , reg : gp21 , asm : "REMV" , typ : "Int64" } , // arg0 / arg1, signed
{ name : "REMVU" , argLength : 2 , reg : gp21 , asm : "REMVU" , typ : "UInt64" } , // arg0 / arg1, unsigned
2021-11-24 17:31:18 +08:00
{ name : "ADDF" , argLength : 2 , reg : fp21 , asm : "ADDF" , commutative : true } , // arg0 + arg1
{ name : "ADDD" , argLength : 2 , reg : fp21 , asm : "ADDD" , commutative : true } , // arg0 + arg1
{ name : "SUBF" , argLength : 2 , reg : fp21 , asm : "SUBF" } , // arg0 - arg1
{ name : "SUBD" , argLength : 2 , reg : fp21 , asm : "SUBD" } , // arg0 - arg1
{ name : "MULF" , argLength : 2 , reg : fp21 , asm : "MULF" , commutative : true } , // arg0 * arg1
{ name : "MULD" , argLength : 2 , reg : fp21 , asm : "MULD" , commutative : true } , // arg0 * arg1
{ name : "DIVF" , argLength : 2 , reg : fp21 , asm : "DIVF" } , // arg0 / arg1
{ name : "DIVD" , argLength : 2 , reg : fp21 , asm : "DIVD" } , // arg0 / arg1
{ name : "AND" , argLength : 2 , reg : gp21 , asm : "AND" , commutative : true } , // arg0 & arg1
{ name : "ANDconst" , argLength : 1 , reg : gp11 , asm : "AND" , aux : "Int64" } , // arg0 & auxInt
{ name : "OR" , argLength : 2 , reg : gp21 , asm : "OR" , commutative : true } , // arg0 | arg1
{ name : "ORconst" , argLength : 1 , reg : gp11 , asm : "OR" , aux : "Int64" } , // arg0 | auxInt
{ name : "XOR" , argLength : 2 , reg : gp21 , asm : "XOR" , commutative : true , typ : "UInt64" } , // arg0 ^ arg1
{ name : "XORconst" , argLength : 1 , reg : gp11 , asm : "XOR" , aux : "Int64" , typ : "UInt64" } , // arg0 ^ auxInt
{ name : "NOR" , argLength : 2 , reg : gp21 , asm : "NOR" , commutative : true } , // ^(arg0 | arg1)
{ name : "NORconst" , argLength : 1 , reg : gp11 , asm : "NOR" , aux : "Int64" } , // ^(arg0 | auxInt)
2025-05-20 10:28:17 +08:00
{ name : "ANDN" , argLength : 2 , reg : gp21 , asm : "ANDN" } , // arg0 & ^arg1
{ name : "ORN" , argLength : 2 , reg : gp21 , asm : "ORN" } , // arg0 | ^arg1
2021-11-24 17:31:18 +08:00
2024-11-05 15:30:45 +08:00
{ name : "FMADDF" , argLength : 3 , reg : fp31 , asm : "FMADDF" , commutative : true , typ : "Float32" } , // (arg0 * arg1) + arg2
{ name : "FMADDD" , argLength : 3 , reg : fp31 , asm : "FMADDD" , commutative : true , typ : "Float64" } , // (arg0 * arg1) + arg2
{ name : "FMSUBF" , argLength : 3 , reg : fp31 , asm : "FMSUBF" , commutative : true , typ : "Float32" } , // (arg0 * arg1) - arg2
{ name : "FMSUBD" , argLength : 3 , reg : fp31 , asm : "FMSUBD" , commutative : true , typ : "Float64" } , // (arg0 * arg1) - arg2
{ name : "FNMADDF" , argLength : 3 , reg : fp31 , asm : "FNMADDF" , commutative : true , typ : "Float32" } , // -((arg0 * arg1) + arg2)
{ name : "FNMADDD" , argLength : 3 , reg : fp31 , asm : "FNMADDD" , commutative : true , typ : "Float64" } , // -((arg0 * arg1) + arg2)
{ name : "FNMSUBF" , argLength : 3 , reg : fp31 , asm : "FNMSUBF" , commutative : true , typ : "Float32" } , // -((arg0 * arg1) - arg2)
{ name : "FNMSUBD" , argLength : 3 , reg : fp31 , asm : "FNMSUBD" , commutative : true , typ : "Float64" } , // -((arg0 * arg1) - arg2)
2024-08-07 21:25:41 +02:00
{ name : "FMINF" , argLength : 2 , reg : fp21 , resultNotInArgs : true , asm : "FMINF" , commutative : true , typ : "Float32" } , // min(arg0, arg1), float32
{ name : "FMIND" , argLength : 2 , reg : fp21 , resultNotInArgs : true , asm : "FMIND" , commutative : true , typ : "Float64" } , // min(arg0, arg1), float64
{ name : "FMAXF" , argLength : 2 , reg : fp21 , resultNotInArgs : true , asm : "FMAXF" , commutative : true , typ : "Float32" } , // max(arg0, arg1), float32
{ name : "FMAXD" , argLength : 2 , reg : fp21 , resultNotInArgs : true , asm : "FMAXD" , commutative : true , typ : "Float64" } , // max(arg0, arg1), float64
2024-04-10 11:48:11 +08:00
2024-10-18 16:31:29 +08:00
{ name : "MASKEQZ" , argLength : 2 , reg : gp21 , asm : "MASKEQZ" } , // returns 0 if arg1 == 0, otherwise returns arg0
{ name : "MASKNEZ" , argLength : 2 , reg : gp21 , asm : "MASKNEZ" } , // returns 0 if arg1 != 0, otherwise returns arg0
2024-04-01 17:13:42 +08:00
{ name : "FCOPYSGD" , argLength : 2 , reg : fp21 , asm : "FCOPYSGD" } , // float64
2021-11-24 17:31:18 +08:00
// shifts
2024-11-14 11:35:39 +08:00
{ name : "SLL" , argLength : 2 , reg : gp21 , asm : "SLL" } , // arg0 << arg1, shift amount is mod 32
2022-08-09 23:53:37 +08:00
{ name : "SLLV" , argLength : 2 , reg : gp21 , asm : "SLLV" } , // arg0 << arg1, shift amount is mod 64
2024-11-14 11:35:39 +08:00
{ name : "SLLconst" , argLength : 1 , reg : gp11 , asm : "SLL" , aux : "Int64" } , // arg0 << auxInt, auxInt should be in the range 0 to 31.
2022-08-09 23:53:37 +08:00
{ name : "SLLVconst" , argLength : 1 , reg : gp11 , asm : "SLLV" , aux : "Int64" } , // arg0 << auxInt
2024-11-14 11:35:39 +08:00
{ name : "SRL" , argLength : 2 , reg : gp21 , asm : "SRL" } , // arg0 >> arg1, shift amount is mod 32
2022-08-09 23:53:37 +08:00
{ name : "SRLV" , argLength : 2 , reg : gp21 , asm : "SRLV" } , // arg0 >> arg1, unsigned, shift amount is mod 64
2024-11-14 11:35:39 +08:00
{ name : "SRLconst" , argLength : 1 , reg : gp11 , asm : "SRL" , aux : "Int64" } , // arg0 >> auxInt, auxInt should be in the range 0 to 31.
2022-08-09 23:53:37 +08:00
{ name : "SRLVconst" , argLength : 1 , reg : gp11 , asm : "SRLV" , aux : "Int64" } , // arg0 >> auxInt, unsigned
2024-11-14 11:35:39 +08:00
{ name : "SRA" , argLength : 2 , reg : gp21 , asm : "SRA" } , // arg0 >> arg1, shift amount is mod 32
2022-08-09 23:53:37 +08:00
{ name : "SRAV" , argLength : 2 , reg : gp21 , asm : "SRAV" } , // arg0 >> arg1, signed, shift amount is mod 64
2024-11-14 11:35:39 +08:00
{ name : "SRAconst" , argLength : 1 , reg : gp11 , asm : "SRA" , aux : "Int64" } , // arg0 >> auxInt, signed, auxInt should be in the range 0 to 31.
2022-08-09 23:53:37 +08:00
{ name : "SRAVconst" , argLength : 1 , reg : gp11 , asm : "SRAV" , aux : "Int64" } , // arg0 >> auxInt, signed
{ name : "ROTR" , argLength : 2 , reg : gp21 , asm : "ROTR" } , // arg0 right rotate by (arg1 mod 32) bits
{ name : "ROTRV" , argLength : 2 , reg : gp21 , asm : "ROTRV" } , // arg0 right rotate by (arg1 mod 64) bits
{ name : "ROTRconst" , argLength : 1 , reg : gp11 , asm : "ROTR" , aux : "Int64" } , // uint32(arg0) right rotate by auxInt bits, auxInt should be in the range 0 to 31.
{ name : "ROTRVconst" , argLength : 1 , reg : gp11 , asm : "ROTRV" , aux : "Int64" } , // arg0 right rotate by auxInt bits, auxInt should be in the range 0 to 63.
2021-11-24 17:31:18 +08:00
// comparisons
{ name : "SGT" , argLength : 2 , reg : gp21 , asm : "SGT" , typ : "Bool" } , // 1 if arg0 > arg1 (signed), 0 otherwise
{ name : "SGTconst" , argLength : 1 , reg : gp11 , asm : "SGT" , aux : "Int64" , typ : "Bool" } , // 1 if auxInt > arg0 (signed), 0 otherwise
{ name : "SGTU" , argLength : 2 , reg : gp21 , asm : "SGTU" , typ : "Bool" } , // 1 if arg0 > arg1 (unsigned), 0 otherwise
{ name : "SGTUconst" , argLength : 1 , reg : gp11 , asm : "SGTU" , aux : "Int64" , typ : "Bool" } , // 1 if auxInt > arg0 (unsigned), 0 otherwise
{ name : "CMPEQF" , argLength : 2 , reg : fp2flags , asm : "CMPEQF" , typ : "Flags" } , // flags=true if arg0 = arg1, float32
{ name : "CMPEQD" , argLength : 2 , reg : fp2flags , asm : "CMPEQD" , typ : "Flags" } , // flags=true if arg0 = arg1, float64
{ name : "CMPGEF" , argLength : 2 , reg : fp2flags , asm : "CMPGEF" , typ : "Flags" } , // flags=true if arg0 >= arg1, float32
{ name : "CMPGED" , argLength : 2 , reg : fp2flags , asm : "CMPGED" , typ : "Flags" } , // flags=true if arg0 >= arg1, float64
{ name : "CMPGTF" , argLength : 2 , reg : fp2flags , asm : "CMPGTF" , typ : "Flags" } , // flags=true if arg0 > arg1, float32
{ name : "CMPGTD" , argLength : 2 , reg : fp2flags , asm : "CMPGTD" , typ : "Flags" } , // flags=true if arg0 > arg1, float64
2024-08-14 17:11:08 +08:00
// bitfield ops
// for bstrpick.w msbw is auxInt>>5, lsbw is auxInt&0x1f
// for bstrpick.d msbd is auxInt>>6, lsbd is auxInt&0x3f
{ name : "BSTRPICKW" , argLength : 1 , reg : gp11 , asm : "BSTRPICKW" , aux : "Int64" } ,
{ name : "BSTRPICKV" , argLength : 1 , reg : gp11 , asm : "BSTRPICKV" , aux : "Int64" } ,
2021-11-24 17:31:18 +08:00
// moves
{ name : "MOVVconst" , argLength : 0 , reg : gp01 , aux : "Int64" , asm : "MOVV" , typ : "UInt64" , rematerializeable : true } , // auxint
{ name : "MOVFconst" , argLength : 0 , reg : fp01 , aux : "Float64" , asm : "MOVF" , typ : "Float32" , rematerializeable : true } , // auxint as 64-bit float, convert to 32-bit float
{ name : "MOVDconst" , argLength : 0 , reg : fp01 , aux : "Float64" , asm : "MOVD" , typ : "Float64" , rematerializeable : true } , // auxint as 64-bit float
{ name : "MOVVaddr" , argLength : 1 , reg : regInfo { inputs : [ ] regMask { buildReg ( "SP" ) | buildReg ( "SB" ) } , outputs : [ ] regMask { gp } } , aux : "SymOff" , asm : "MOVV" , rematerializeable : true , symEffect : "Addr" } , // arg0 + auxInt + aux.(*gc.Sym), arg0=SP/SB
{ name : "MOVBload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVB" , typ : "Int8" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVBUload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVBU" , typ : "UInt8" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVHload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVH" , typ : "Int16" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVHUload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVHU" , typ : "UInt16" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVWload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVW" , typ : "Int32" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVWUload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVWU" , typ : "UInt32" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVVload" , argLength : 2 , reg : gpload , aux : "SymOff" , asm : "MOVV" , typ : "UInt64" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVFload" , argLength : 2 , reg : fpload , aux : "SymOff" , asm : "MOVF" , typ : "Float32" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
{ name : "MOVDload" , argLength : 2 , reg : fpload , aux : "SymOff" , asm : "MOVD" , typ : "Float64" , faultOnNilArg0 : true , symEffect : "Read" } , // load from arg0 + auxInt + aux. arg1=mem.
2024-09-26 14:17:17 +08:00
// register indexed load
{ name : "MOVVloadidx" , argLength : 3 , reg : gp2load , asm : "MOVV" , typ : "UInt64" } , // load 64-bit dword from arg0 + arg1, arg2 = mem.
{ name : "MOVWloadidx" , argLength : 3 , reg : gp2load , asm : "MOVW" , typ : "Int32" } , // load 32-bit word from arg0 + arg1, sign-extended to 64-bit, arg2=mem.
{ name : "MOVWUloadidx" , argLength : 3 , reg : gp2load , asm : "MOVWU" , typ : "UInt32" } , // load 32-bit word from arg0 + arg1, zero-extended to 64-bit, arg2=mem.
{ name : "MOVHloadidx" , argLength : 3 , reg : gp2load , asm : "MOVH" , typ : "Int16" } , // load 16-bit word from arg0 + arg1, sign-extended to 64-bit, arg2=mem.
{ name : "MOVHUloadidx" , argLength : 3 , reg : gp2load , asm : "MOVHU" , typ : "UInt16" } , // load 16-bit word from arg0 + arg1, zero-extended to 64-bit, arg2=mem.
{ name : "MOVBloadidx" , argLength : 3 , reg : gp2load , asm : "MOVB" , typ : "Int8" } , // load 8-bit word from arg0 + arg1, sign-extended to 64-bit, arg2=mem.
{ name : "MOVBUloadidx" , argLength : 3 , reg : gp2load , asm : "MOVBU" , typ : "UInt8" } , // load 8-bit word from arg0 + arg1, zero-extended to 64-bit, arg2=mem.
{ name : "MOVFloadidx" , argLength : 3 , reg : fp2load , asm : "MOVF" , typ : "Float32" } , // load 32-bit float from arg0 + arg1, arg2=mem.
{ name : "MOVDloadidx" , argLength : 3 , reg : fp2load , asm : "MOVD" , typ : "Float64" } , // load 64-bit float from arg0 + arg1, arg2=mem.
2021-11-24 17:31:18 +08:00
{ name : "MOVBstore" , argLength : 3 , reg : gpstore , aux : "SymOff" , asm : "MOVB" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 1 byte of arg1 to arg0 + auxInt + aux. arg2=mem.
{ name : "MOVHstore" , argLength : 3 , reg : gpstore , aux : "SymOff" , asm : "MOVH" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 2 bytes of arg1 to arg0 + auxInt + aux. arg2=mem.
{ name : "MOVWstore" , argLength : 3 , reg : gpstore , aux : "SymOff" , asm : "MOVW" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 4 bytes of arg1 to arg0 + auxInt + aux. arg2=mem.
{ name : "MOVVstore" , argLength : 3 , reg : gpstore , aux : "SymOff" , asm : "MOVV" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 8 bytes of arg1 to arg0 + auxInt + aux. arg2=mem.
{ name : "MOVFstore" , argLength : 3 , reg : fpstore , aux : "SymOff" , asm : "MOVF" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 4 bytes of arg1 to arg0 + auxInt + aux. arg2=mem.
{ name : "MOVDstore" , argLength : 3 , reg : fpstore , aux : "SymOff" , asm : "MOVD" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 8 bytes of arg1 to arg0 + auxInt + aux. arg2=mem.
2024-09-26 14:17:17 +08:00
// register indexed store
{ name : "MOVBstoreidx" , argLength : 4 , reg : gpstore2 , asm : "MOVB" , typ : "Mem" } , // store 1 byte of arg2 to arg0 + arg1, arg3 = mem.
{ name : "MOVHstoreidx" , argLength : 4 , reg : gpstore2 , asm : "MOVH" , typ : "Mem" } , // store 2 bytes of arg2 to arg0 + arg1, arg3 = mem.
{ name : "MOVWstoreidx" , argLength : 4 , reg : gpstore2 , asm : "MOVW" , typ : "Mem" } , // store 4 bytes of arg2 to arg0 + arg1, arg3 = mem.
{ name : "MOVVstoreidx" , argLength : 4 , reg : gpstore2 , asm : "MOVV" , typ : "Mem" } , // store 8 bytes of arg2 to arg0 + arg1, arg3 = mem.
{ name : "MOVFstoreidx" , argLength : 4 , reg : fpstore2 , asm : "MOVF" , typ : "Mem" } , // store 32-bit float of arg2 to arg0 + arg1, arg3=mem.
{ name : "MOVDstoreidx" , argLength : 4 , reg : fpstore2 , asm : "MOVD" , typ : "Mem" } , // store 64-bit float of arg2 to arg0 + arg1, arg3=mem.
2021-11-24 17:31:18 +08:00
{ name : "MOVBstorezero" , argLength : 2 , reg : gpstore0 , aux : "SymOff" , asm : "MOVB" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 1 byte of zero to arg0 + auxInt + aux. arg1=mem.
{ name : "MOVHstorezero" , argLength : 2 , reg : gpstore0 , aux : "SymOff" , asm : "MOVH" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 2 bytes of zero to arg0 + auxInt + aux. arg1=mem.
{ name : "MOVWstorezero" , argLength : 2 , reg : gpstore0 , aux : "SymOff" , asm : "MOVW" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 4 bytes of zero to arg0 + auxInt + aux. arg1=mem.
{ name : "MOVVstorezero" , argLength : 2 , reg : gpstore0 , aux : "SymOff" , asm : "MOVV" , typ : "Mem" , faultOnNilArg0 : true , symEffect : "Write" } , // store 8 bytes of zero to arg0 + auxInt + aux. ar12=mem.
2024-09-26 14:17:17 +08:00
// register indexed store zero
{ name : "MOVBstorezeroidx" , argLength : 3 , reg : gpstore , asm : "MOVB" , typ : "Mem" } , // store 1 byte of zero to arg0 + arg1, arg2 = mem.
{ name : "MOVHstorezeroidx" , argLength : 3 , reg : gpstore , asm : "MOVH" , typ : "Mem" } , // store 2 bytes of zero to arg0 + arg1, arg2 = mem.
{ name : "MOVWstorezeroidx" , argLength : 3 , reg : gpstore , asm : "MOVW" , typ : "Mem" } , // store 4 bytes of zero to arg0 + arg1, arg2 = mem.
{ name : "MOVVstorezeroidx" , argLength : 3 , reg : gpstore , asm : "MOVV" , typ : "Mem" } , // store 8 bytes of zero to arg0 + arg1, arg2 = mem.
2024-08-12 16:41:11 +08:00
// moves (no conversion)
{ name : "MOVWfpgp" , argLength : 1 , reg : fpgp , asm : "MOVW" } , // move float32 to int32 (no conversion).
{ name : "MOVWgpfp" , argLength : 1 , reg : gpfp , asm : "MOVW" } , // move int32 to float32 (no conversion).
{ name : "MOVVfpgp" , argLength : 1 , reg : fpgp , asm : "MOVV" } , // move float64 to int64 (no conversion).
{ name : "MOVVgpfp" , argLength : 1 , reg : gpfp , asm : "MOVV" } , // move int64 to float64 (no conversion).
2021-11-24 17:31:18 +08:00
// conversions
{ name : "MOVBreg" , argLength : 1 , reg : gp11 , asm : "MOVB" } , // move from arg0, sign-extended from byte
{ name : "MOVBUreg" , argLength : 1 , reg : gp11 , asm : "MOVBU" } , // move from arg0, unsign-extended from byte
{ name : "MOVHreg" , argLength : 1 , reg : gp11 , asm : "MOVH" } , // move from arg0, sign-extended from half
{ name : "MOVHUreg" , argLength : 1 , reg : gp11 , asm : "MOVHU" } , // move from arg0, unsign-extended from half
{ name : "MOVWreg" , argLength : 1 , reg : gp11 , asm : "MOVW" } , // move from arg0, sign-extended from word
{ name : "MOVWUreg" , argLength : 1 , reg : gp11 , asm : "MOVWU" } , // move from arg0, unsign-extended from word
{ name : "MOVVreg" , argLength : 1 , reg : gp11 , asm : "MOVV" } , // move from arg0
{ name : "MOVVnop" , argLength : 1 , reg : regInfo { inputs : [ ] regMask { gp } , outputs : [ ] regMask { gp } } , resultInArg0 : true } , // nop, return arg0 in same register
{ name : "MOVWF" , argLength : 1 , reg : fp11 , asm : "MOVWF" } , // int32 -> float32
{ name : "MOVWD" , argLength : 1 , reg : fp11 , asm : "MOVWD" } , // int32 -> float64
{ name : "MOVVF" , argLength : 1 , reg : fp11 , asm : "MOVVF" } , // int64 -> float32
{ name : "MOVVD" , argLength : 1 , reg : fp11 , asm : "MOVVD" } , // int64 -> float64
{ name : "TRUNCFW" , argLength : 1 , reg : fp11 , asm : "TRUNCFW" } , // float32 -> int32
{ name : "TRUNCDW" , argLength : 1 , reg : fp11 , asm : "TRUNCDW" } , // float64 -> int32
{ name : "TRUNCFV" , argLength : 1 , reg : fp11 , asm : "TRUNCFV" } , // float32 -> int64
{ name : "TRUNCDV" , argLength : 1 , reg : fp11 , asm : "TRUNCDV" } , // float64 -> int64
{ name : "MOVFD" , argLength : 1 , reg : fp11 , asm : "MOVFD" } , // float32 -> float64
{ name : "MOVDF" , argLength : 1 , reg : fp11 , asm : "MOVDF" } , // float64 -> float32
2024-11-05 15:30:45 +08:00
// Round ops to block fused-multiply-add extraction.
{ name : "LoweredRound32F" , argLength : 1 , reg : fp11 , resultInArg0 : true } ,
{ name : "LoweredRound64F" , argLength : 1 , reg : fp11 , resultInArg0 : true } ,
2021-11-24 17:31:18 +08:00
// function calls
2023-08-15 20:23:46 +08:00
{ name : "CALLstatic" , argLength : - 1 , reg : regInfo { clobbers : callerSave } , aux : "CallOff" , clobberFlags : true , call : true } , // call static function aux.(*obj.LSym). last arg=mem, auxint=argsize, returns mem
{ name : "CALLtail" , argLength : - 1 , reg : regInfo { clobbers : callerSave } , aux : "CallOff" , clobberFlags : true , call : true , tailCall : true } , // tail call static function aux.(*obj.LSym). last arg=mem, auxint=argsize, returns mem
{ name : "CALLclosure" , argLength : - 1 , reg : regInfo { inputs : [ ] regMask { gpsp , buildReg ( "R29" ) , 0 } , clobbers : callerSave } , aux : "CallOff" , clobberFlags : true , call : true } , // call function via closure. arg0=codeptr, arg1=closure, last arg=mem, auxint=argsize, returns mem
{ name : "CALLinter" , argLength : - 1 , reg : regInfo { inputs : [ ] regMask { gp } , clobbers : callerSave } , aux : "CallOff" , clobberFlags : true , call : true } , // call fn by pointer. arg0=codeptr, last arg=mem, auxint=argsize, returns mem
2021-11-24 17:31:18 +08:00
// duffzero
// arg0 = address of memory to zero
// arg1 = mem
// auxint = offset into duffzero code to start executing
// returns mem
2023-08-17 03:58:10 +08:00
// R20 aka loong64.REGRT1 changed as side effect
2021-11-24 17:31:18 +08:00
{
name : "DUFFZERO" ,
aux : "Int64" ,
argLength : 2 ,
reg : regInfo {
2023-08-17 03:58:10 +08:00
inputs : [ ] regMask { buildReg ( "R20" ) } ,
clobbers : buildReg ( "R20 R1" ) ,
2021-11-24 17:31:18 +08:00
} ,
2023-04-21 11:08:09 +08:00
typ : "Mem" ,
2021-11-24 17:31:18 +08:00
faultOnNilArg0 : true ,
} ,
// duffcopy
2023-08-17 03:58:10 +08:00
// arg0 = address of dst memory (in R21, changed as side effect)
// arg1 = address of src memory (in R20, changed as side effect)
2021-11-24 17:31:18 +08:00
// arg2 = mem
// auxint = offset into duffcopy code to start executing
// returns mem
{
name : "DUFFCOPY" ,
aux : "Int64" ,
argLength : 3 ,
reg : regInfo {
2023-08-17 03:58:10 +08:00
inputs : [ ] regMask { buildReg ( "R21" ) , buildReg ( "R20" ) } ,
clobbers : buildReg ( "R20 R21 R1" ) ,
2021-11-24 17:31:18 +08:00
} ,
2023-04-21 11:08:09 +08:00
typ : "Mem" ,
2021-11-24 17:31:18 +08:00
faultOnNilArg0 : true ,
faultOnNilArg1 : true ,
} ,
// large or unaligned zeroing
2023-08-17 03:58:10 +08:00
// arg0 = address of memory to zero (in R20, changed as side effect)
2021-11-24 17:31:18 +08:00
// arg1 = address of the last element to zero
// arg2 = mem
// auxint = alignment
// returns mem
2023-08-17 03:58:10 +08:00
// MOVx R0, (R20)
// ADDV $sz, R20
// BGEU Rarg1, R20, -2(PC)
2021-11-24 17:31:18 +08:00
{
name : "LoweredZero" ,
aux : "Int64" ,
argLength : 3 ,
reg : regInfo {
2023-08-17 03:58:10 +08:00
inputs : [ ] regMask { buildReg ( "R20" ) , gp } ,
clobbers : buildReg ( "R20" ) ,
2021-11-24 17:31:18 +08:00
} ,
2023-04-25 03:27:23 +08:00
typ : "Mem" ,
2021-11-24 17:31:18 +08:00
faultOnNilArg0 : true ,
} ,
// large or unaligned move
2023-08-17 03:58:10 +08:00
// arg0 = address of dst memory (in R21, changed as side effect)
// arg1 = address of src memory (in R20, changed as side effect)
2021-11-24 17:31:18 +08:00
// arg2 = address of the last element of src
// arg3 = mem
// auxint = alignment
// returns mem
2023-08-17 03:58:10 +08:00
// MOVx (R20), Rtmp
// MOVx Rtmp, (R21)
2023-04-25 03:27:23 +08:00
// ADDV $sz, R20
2023-08-17 03:58:10 +08:00
// ADDV $sz, R21
// BGEU Rarg2, R20, -4(PC)
2021-11-24 17:31:18 +08:00
{
name : "LoweredMove" ,
aux : "Int64" ,
argLength : 4 ,
reg : regInfo {
2023-08-17 03:58:10 +08:00
inputs : [ ] regMask { buildReg ( "R21" ) , buildReg ( "R20" ) , gp } ,
clobbers : buildReg ( "R20 R21" ) ,
2021-11-24 17:31:18 +08:00
} ,
2023-04-25 03:27:23 +08:00
typ : "Mem" ,
2021-11-24 17:31:18 +08:00
faultOnNilArg0 : true ,
faultOnNilArg1 : true ,
} ,
// atomic loads.
// load from arg0. arg1=mem.
// returns <value,memory> so they can be properly ordered with other loads.
{ name : "LoweredAtomicLoad8" , argLength : 2 , reg : gpload , faultOnNilArg0 : true } ,
{ name : "LoweredAtomicLoad32" , argLength : 2 , reg : gpload , faultOnNilArg0 : true } ,
{ name : "LoweredAtomicLoad64" , argLength : 2 , reg : gpload , faultOnNilArg0 : true } ,
// atomic stores.
// store arg1 to arg0. arg2=mem. returns memory.
{ name : "LoweredAtomicStore8" , argLength : 3 , reg : gpstore , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicStore32" , argLength : 3 , reg : gpstore , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicStore64" , argLength : 3 , reg : gpstore , faultOnNilArg0 : true , hasSideEffects : true } ,
cmd/compiler,internal/runtime/atomic: optimize Store{64,32,8} on loong64
On Loong64, AMSWAPDB{W,V} instructions are supported by default, and AMSWAPDB{B,H} [1]
is a new instruction added by LA664(Loongson 3A6000) and later microarchitectures.
Therefore, AMSWAPDB{W,V} (full barrier) is used to implement AtomicStore{32,64}, and
the traditional MOVB or the new AMSWAPDBB is used to implement AtomicStore8 according
to the CPU feature.
The StoreRelease barrier on Loong64 is "dbar 0x12", but it is still necessary to
ensure consistency in the order of Store/Load [2].
LoweredAtomicStorezero{32,64} was removed because on loong64 the constant "0" uses
the R0 register, and there is no performance difference between the implementations
of LoweredAtomicStorezero{32,64} and LoweredAtomicStore{32,64}.
goos: linux
goarch: loong64
pkg: internal/runtime/atomic
cpu: Loongson-3A5000-HV @ 2500.00MHz
| bench.old | bench.new |
| sec/op | sec/op vs base |
AtomicStore64 19.61n ± 0% 13.61n ± 0% -30.60% (p=0.000 n=20)
AtomicStore64-2 19.61n ± 0% 13.61n ± 0% -30.57% (p=0.000 n=20)
AtomicStore64-4 19.62n ± 0% 13.61n ± 0% -30.63% (p=0.000 n=20)
AtomicStore 19.61n ± 0% 13.61n ± 0% -30.60% (p=0.000 n=20)
AtomicStore-2 19.62n ± 0% 13.61n ± 0% -30.63% (p=0.000 n=20)
AtomicStore-4 19.62n ± 0% 13.62n ± 0% -30.58% (p=0.000 n=20)
AtomicStore8 19.61n ± 0% 20.01n ± 0% +2.04% (p=0.000 n=20)
AtomicStore8-2 19.62n ± 0% 20.02n ± 0% +2.01% (p=0.000 n=20)
AtomicStore8-4 19.61n ± 0% 20.02n ± 0% +2.09% (p=0.000 n=20)
geomean 19.61n 15.48n -21.08%
goos: linux
goarch: loong64
pkg: internal/runtime/atomic
cpu: Loongson-3A6000 @ 2500.00MHz
| bench.old | bench.new |
| sec/op | sec/op vs base |
AtomicStore64 18.03n ± 0% 12.81n ± 0% -28.93% (p=0.000 n=20)
AtomicStore64-2 18.02n ± 0% 12.81n ± 0% -28.91% (p=0.000 n=20)
AtomicStore64-4 18.01n ± 0% 12.81n ± 0% -28.87% (p=0.000 n=20)
AtomicStore 18.02n ± 0% 12.81n ± 0% -28.91% (p=0.000 n=20)
AtomicStore-2 18.01n ± 0% 12.81n ± 0% -28.87% (p=0.000 n=20)
AtomicStore-4 18.01n ± 0% 12.81n ± 0% -28.87% (p=0.000 n=20)
AtomicStore8 18.01n ± 0% 12.81n ± 0% -28.87% (p=0.000 n=20)
AtomicStore8-2 18.01n ± 0% 12.81n ± 0% -28.87% (p=0.000 n=20)
AtomicStore8-4 18.01n ± 0% 12.81n ± 0% -28.87% (p=0.000 n=20)
geomean 18.01n 12.81n -28.89%
[1]: https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html
[2]: https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=gcc/config/loongarch/sync.md
Change-Id: I4ae5e8dd0e6f026129b6e503990a763ed40c6097
Reviewed-on: https://go-review.googlesource.com/c/go/+/581356
Reviewed-by: sophie zhao <zhaoxiaolin@loongson.cn>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Qiqi Huang <huangqiqi@loongson.cn>
Reviewed-by: Meidan Li <limeidan@loongson.cn>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: David Chase <drchase@google.com>
2024-09-13 18:47:56 +08:00
{ name : "LoweredAtomicStore8Variant" , argLength : 3 , reg : gpstore , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicStore32Variant" , argLength : 3 , reg : gpstore , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicStore64Variant" , argLength : 3 , reg : gpstore , faultOnNilArg0 : true , hasSideEffects : true } ,
2021-11-24 17:31:18 +08:00
// atomic exchange.
// store arg1 to arg0. arg2=mem. returns <old content of *arg0, memory>.
2023-04-01 08:49:58 +08:00
{ name : "LoweredAtomicExchange32" , argLength : 3 , reg : gpxchg , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicExchange64" , argLength : 3 , reg : gpxchg , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
2021-11-24 17:31:18 +08:00
2024-10-30 19:11:49 +08:00
// atomic exchange variant.
// store arg1 to arg0. arg2=mem. returns <old content of *arg0, memory>. auxint must be zero.
// AMSWAPDBB Rarg1, (Rarg0), Rout
{ name : "LoweredAtomicExchange8Variant" , argLength : 3 , reg : gpxchg , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
2021-11-24 17:31:18 +08:00
// atomic add.
// *arg0 += arg1. arg2=mem. returns <new content of *arg0, memory>.
2023-04-03 12:11:46 +08:00
{ name : "LoweredAtomicAdd32" , argLength : 3 , reg : gpxchg , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicAdd64" , argLength : 3 , reg : gpxchg , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
2021-11-24 17:31:18 +08:00
// atomic compare and swap.
// arg0 = pointer, arg1 = old value, arg2 = new value, arg3 = memory.
// if *arg0 == arg1 {
// *arg0 = arg2
// return (true, memory)
// } else {
// return (false, memory)
// }
// MOVV $0, Rout
2024-09-20 11:06:18 +08:00
// DBAR 0x14
2021-11-24 17:31:18 +08:00
// LL (Rarg0), Rtmp
// BNE Rtmp, Rarg1, 4(PC)
// MOVV Rarg2, Rout
// SC Rout, (Rarg0)
// BEQ Rout, -4(PC)
2024-09-20 11:06:18 +08:00
// DBAR 0x12
2021-11-24 17:31:18 +08:00
{ name : "LoweredAtomicCas32" , argLength : 4 , reg : gpcas , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true , unsafePoint : true } ,
{ name : "LoweredAtomicCas64" , argLength : 4 , reg : gpcas , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true , unsafePoint : true } ,
2024-09-20 11:06:18 +08:00
// atomic compare and swap variant.
// arg0 = pointer, arg1 = old value, arg2 = new value, arg3 = memory. auxint must be zero.
// if *arg0 == arg1 {
// *arg0 = arg2
// return (true, memory)
// } else {
// return (false, memory)
// }
// MOVV $0, Rout
// MOVV Rarg1, Rtmp
// AMCASDBx Rarg2, (Rarg0), Rtmp
// BNE Rarg1, Rtmp, 2(PC)
// MOVV $1, Rout
// NOP
{ name : "LoweredAtomicCas64Variant" , argLength : 4 , reg : gpcas , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true , unsafePoint : true } ,
{ name : "LoweredAtomicCas32Variant" , argLength : 4 , reg : gpcas , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true , unsafePoint : true } ,
2021-11-24 17:31:18 +08:00
cmd/compiler,internal/runtime/atomic: optimize And{64,32,8} and Or{64,32,8} on loong64
Use loong64's atomic operation instruction AMANDDB{V,W,W} (full barrier) to implement
And{64,32,8}, AMORDB{V,W,W} (full barrier) to implement Or{64,32,8}.
Intrinsify And{64,32,8} and Or{64,32,8}, And this CL alias all of the And/Or operations
into sync/atomic package.
goos: linux
goarch: loong64
pkg: internal/runtime/atomic
cpu: Loongson-3A6000-HV @ 2500.00MHz
| bench.old | bench.new |
| sec/op | sec/op vs base |
And32 27.73n ± 0% 10.81n ± 0% -61.02% (p=0.000 n=20)
And32Parallel 28.96n ± 0% 12.41n ± 0% -57.15% (p=0.000 n=20)
And64 27.73n ± 0% 10.81n ± 0% -61.02% (p=0.000 n=20)
And64Parallel 28.96n ± 0% 12.41n ± 0% -57.15% (p=0.000 n=20)
Or32 27.62n ± 0% 10.81n ± 0% -60.86% (p=0.000 n=20)
Or32Parallel 28.96n ± 0% 12.41n ± 0% -57.15% (p=0.000 n=20)
Or64 27.62n ± 0% 10.81n ± 0% -60.86% (p=0.000 n=20)
Or64Parallel 28.97n ± 0% 12.41n ± 0% -57.16% (p=0.000 n=20)
And8 29.15n ± 0% 13.21n ± 0% -54.68% (p=0.000 n=20)
And 27.71n ± 0% 12.82n ± 0% -53.74% (p=0.000 n=20)
And8Parallel 28.99n ± 0% 14.46n ± 0% -50.12% (p=0.000 n=20)
AndParallel 29.12n ± 0% 14.42n ± 0% -50.48% (p=0.000 n=20)
Or8 28.31n ± 0% 12.81n ± 0% -54.75% (p=0.000 n=20)
Or 27.72n ± 0% 12.81n ± 0% -53.79% (p=0.000 n=20)
Or8Parallel 29.03n ± 0% 14.62n ± 0% -49.64% (p=0.000 n=20)
OrParallel 29.12n ± 0% 14.42n ± 0% -50.49% (p=0.000 n=20)
geomean 28.47n 12.58n -55.80%
goos: linux
goarch: loong64
pkg: internal/runtime/atomic
cpu: Loongson-3A5000 @ 2500.00MHz
| bench.old | bench.new |
| sec/op | sec/op vs base |
And32 30.02n ± 0% 14.81n ± 0% -50.67% (p=0.000 n=20)
And32Parallel 30.83n ± 0% 15.61n ± 0% -49.37% (p=0.000 n=20)
And64 30.02n ± 0% 14.81n ± 0% -50.67% (p=0.000 n=20)
And64Parallel 30.83n ± 0% 15.61n ± 0% -49.37% (p=0.000 n=20)
And8 30.42n ± 0% 14.41n ± 0% -52.63% (p=0.000 n=20)
And 30.02n ± 0% 13.61n ± 0% -54.66% (p=0.000 n=20)
And8Parallel 31.23n ± 0% 15.21n ± 0% -51.30% (p=0.000 n=20)
AndParallel 30.83n ± 0% 14.41n ± 0% -53.26% (p=0.000 n=20)
Or32 30.02n ± 0% 14.81n ± 0% -50.67% (p=0.000 n=20)
Or32Parallel 30.83n ± 0% 15.61n ± 0% -49.37% (p=0.000 n=20)
Or64 30.02n ± 0% 14.82n ± 0% -50.63% (p=0.000 n=20)
Or64Parallel 30.83n ± 0% 15.61n ± 0% -49.37% (p=0.000 n=20)
Or8 30.02n ± 0% 14.01n ± 0% -53.33% (p=0.000 n=20)
Or 30.02n ± 0% 13.61n ± 0% -54.66% (p=0.000 n=20)
Or8Parallel 30.83n ± 0% 14.81n ± 0% -51.96% (p=0.000 n=20)
OrParallel 30.83n ± 0% 14.41n ± 0% -53.26% (p=0.000 n=20)
geomean 30.47n 14.75n -51.61%
Change-Id: If008ff6a08b51905076f8ddb6e92f8e214d3f7b3
Reviewed-on: https://go-review.googlesource.com/c/go/+/482756
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Qiqi Huang <huangqiqi@loongson.cn>
Reviewed-by: Meidan Li <limeidan@loongson.cn>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: sophie zhao <zhaoxiaolin@loongson.cn>
Reviewed-by: Cherry Mui <cherryyz@google.com>
2024-09-23 11:38:36 +08:00
// Atomic 32 bit AND/OR.
// *arg0 &= (|=) arg1. arg2=mem. returns nil.
{ name : "LoweredAtomicAnd32" , argLength : 3 , reg : gpxchg , asm : "AMANDDBW" , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicOr32" , argLength : 3 , reg : gpxchg , asm : "AMORDBW" , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
// Atomic 32,64 bit AND/OR.
// *arg0 &= (|=) arg1. arg2=mem. returns <old content of *arg0, memory>. auxint must be zero.
{ name : "LoweredAtomicAnd32value" , argLength : 3 , reg : gpxchg , asm : "AMANDDBW" , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicAnd64value" , argLength : 3 , reg : gpxchg , asm : "AMANDDBV" , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicOr32value" , argLength : 3 , reg : gpxchg , asm : "AMORDBW" , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
{ name : "LoweredAtomicOr64value" , argLength : 3 , reg : gpxchg , asm : "AMORDBV" , resultNotInArgs : true , faultOnNilArg0 : true , hasSideEffects : true } ,
2021-11-24 17:31:18 +08:00
// pseudo-ops
{ name : "LoweredNilCheck" , argLength : 2 , reg : regInfo { inputs : [ ] regMask { gpg } } , nilCheck : true , faultOnNilArg0 : true } , // panic if arg0 is nil. arg1=mem.
{ name : "FPFlagTrue" , argLength : 1 , reg : readflags } , // bool, true if FP flag is true
{ name : "FPFlagFalse" , argLength : 1 , reg : readflags } , // bool, true if FP flag is false
// Scheduler ensures LoweredGetClosurePtr occurs only in entry block,
// and sorts it to the very beginning of the block to prevent other
// use of R22 (loong64.REGCTXT, the closure pointer)
{ name : "LoweredGetClosurePtr" , reg : regInfo { outputs : [ ] regMask { buildReg ( "R29" ) } } , zeroWidth : true } ,
2022-11-26 15:03:51 -08:00
// LoweredGetCallerSP returns the SP of the caller of the current function. arg0=mem.
{ name : "LoweredGetCallerSP" , argLength : 1 , reg : gp01 , rematerializeable : true } ,
2021-11-24 17:31:18 +08:00
// LoweredGetCallerPC evaluates to the PC to which its "caller" will return.
2024-09-16 14:07:43 -04:00
// I.e., if f calls g "calls" sys.GetCallerPC,
2021-11-24 17:31:18 +08:00
// the result should be the PC within f that g will return to.
// See runtime/stubs.go for a more detailed discussion.
{ name : "LoweredGetCallerPC" , reg : gp01 , rematerializeable : true } ,
2022-11-01 16:46:43 -07:00
// LoweredWB invokes runtime.gcWriteBarrier. arg0=mem, auxint=# of buffer entries needed
2021-11-24 17:31:18 +08:00
// It saves all GP registers if necessary,
// but clobbers R1 (LR) because it's a call
// and R30 (REGTMP).
2022-11-01 16:46:43 -07:00
// Returns a pointer to a write barrier buffer in R29.
{ name : "LoweredWB" , argLength : 1 , reg : regInfo { clobbers : ( callerSave &^ gpg ) | buildReg ( "R1" ) , outputs : [ ] regMask { buildReg ( "R29" ) } } , clobberFlags : true , aux : "Int64" } ,
2021-11-24 17:31:18 +08:00
2024-09-19 19:50:23 +08:00
// Do data barrier. arg0=memorys
{ name : "LoweredPubBarrier" , argLength : 1 , asm : "DBAR" , hasSideEffects : true } ,
2025-06-18 15:16:59 -07:00
// LoweredPanicBoundsRR takes x and y, two values that caused a bounds check to fail.
// the RC and CR versions are used when one of the arguments is a constant. CC is used
// when both are constant (normally both 0, as prove derives the fact that a [0] bounds
// failure means the length must have also been 0).
// AuxInt contains a report code (see PanicBounds in genericOps.go).
{ name : "LoweredPanicBoundsRR" , argLength : 3 , aux : "Int64" , reg : regInfo { inputs : [ ] regMask { first16 , first16 } } , typ : "Mem" , call : true } , // arg0=x, arg1=y, arg2=mem, returns memory.
{ name : "LoweredPanicBoundsRC" , argLength : 2 , aux : "PanicBoundsC" , reg : regInfo { inputs : [ ] regMask { first16 } } , typ : "Mem" , call : true } , // arg0=x, arg1=mem, returns memory.
{ name : "LoweredPanicBoundsCR" , argLength : 2 , aux : "PanicBoundsC" , reg : regInfo { inputs : [ ] regMask { first16 } } , typ : "Mem" , call : true } , // arg0=y, arg1=mem, returns memory.
{ name : "LoweredPanicBoundsCC" , argLength : 1 , aux : "PanicBoundsCC" , reg : regInfo { } , typ : "Mem" , call : true } , // arg0=mem, returns memory.
2025-05-12 11:01:44 +08:00
// Prefetch instruction
// Do prefetch arg0 address with option aux. arg0=addr, arg1=memory, aux=option.
// Note:
// The aux of PRELDX is actually composed of two values: $hint and $n. bit[4:0]
// is $hint and bit[41:5] is $n.
{ name : "PRELD" , argLength : 2 , aux : "Int64" , reg : preldreg , asm : "PRELD" , hasSideEffects : true } ,
{ name : "PRELDX" , argLength : 2 , aux : "Int64" , reg : preldreg , asm : "PRELDX" , hasSideEffects : true } ,
2025-08-07 11:23:55 +08:00
{ name : "ADDshiftLLV" , argLength : 2 , aux : "Int64" , reg : gp21 , asm : "ALSLV" } , // arg0 + arg1<<auxInt, the value of auxInt should be in the range [1, 4].
2021-11-24 17:31:18 +08:00
}
blocks := [ ] blockData {
2025-08-05 17:14:35 +08:00
{ name : "EQZ" , controls : 1 } , // = 0
{ name : "NEZ" , controls : 1 } , // != 0
2024-07-18 10:17:42 +08:00
{ name : "LTZ" , controls : 1 } , // < 0
{ name : "LEZ" , controls : 1 } , // <= 0
{ name : "GTZ" , controls : 1 } , // > 0
{ name : "GEZ" , controls : 1 } , // >= 0
{ name : "FPT" , controls : 1 } , // FP flag is true
{ name : "FPF" , controls : 1 } , // FP flag is false
{ name : "BEQ" , controls : 2 } , // controls[0] == controls[1]
2025-08-06 10:31:59 +08:00
{ name : "BNE" , controls : 2 } , // controls[0] != controls[1]
2024-07-18 10:17:42 +08:00
{ name : "BGE" , controls : 2 } , // controls[0] >= controls[1]
{ name : "BLT" , controls : 2 } , // controls[0] < controls[1]
{ name : "BGEU" , controls : 2 } , // controls[0] >= controls[1], unsigned
{ name : "BLTU" , controls : 2 } , // controls[0] < controls[1], unsigned
2021-11-24 17:31:18 +08:00
}
archs = append ( archs , arch {
name : "LOONG64" ,
pkg : "cmd/internal/obj/loong64" ,
genfile : "../../loong64/ssa.go" ,
ops : ops ,
blocks : blocks ,
regnames : regNamesLOONG64 ,
// TODO: support register ABI on loong64
2023-08-15 19:54:51 +08:00
ParamIntRegNames : "R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R18 R19" ,
ParamFloatRegNames : "F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15" ,
2021-11-24 17:31:18 +08:00
gpregmask : gp ,
fpregmask : fp ,
framepointerreg : - 1 , // not used
linkreg : int8 ( num [ "R1" ] ) ,
} )
}
