cmd/compile, runtime: use fine-grained FENCE instructions on riscv64

Make use of the newly supported FENCE operands on riscv64 to replace
overly conservative full memory barriers with more precise memory
ordering instructions.

Change-Id: I8e749eee4f2199cca218229e0cf51779fb6003b4
Reviewed-on: https://go-review.googlesource.com/c/go/+/758020
Reviewed-by: Meng Zhuo <mengzhuo1203@gmail.com>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>

src/cmd/compile/internal/riscv64/ssa.go

@@ -610,13 +610,23 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Sym = ir.Syms.PanicBounds
 
 	case ssa.OpRISCV64LoweredAtomicLoad8:
-		s.Prog(riscv.AFENCE)
+		p1 := s.Prog(riscv.AFENCE)
+		p1.From.Type = obj.TYPE_SPECIAL
+		p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
+		p1.To.Type = obj.TYPE_SPECIAL
+		p1.To.Offset = int64(riscv.SPOP_FENCE_RW)
+
 		p := s.Prog(riscv.AMOVBU)
 		p.From.Type = obj.TYPE_MEM
 		p.From.Reg = v.Args[0].Reg()
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = v.Reg0()
-		s.Prog(riscv.AFENCE)
+
+		p2 := s.Prog(riscv.AFENCE)
+		p2.From.Type = obj.TYPE_SPECIAL
+		p2.From.Offset = int64(riscv.SPOP_FENCE_R)
+		p2.To.Type = obj.TYPE_SPECIAL
+		p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
 
 	case ssa.OpRISCV64LoweredAtomicLoad32, ssa.OpRISCV64LoweredAtomicLoad64:
 		as := riscv.ALRW
@@ -630,13 +640,23 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Reg = v.Reg0()
 
 	case ssa.OpRISCV64LoweredAtomicStore8:
-		s.Prog(riscv.AFENCE)
+		p1 := s.Prog(riscv.AFENCE)
+		p1.From.Type = obj.TYPE_SPECIAL
+		p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
+		p1.To.Type = obj.TYPE_SPECIAL
+		p1.To.Offset = int64(riscv.SPOP_FENCE_W)
+
 		p := s.Prog(riscv.AMOVB)
 		p.From.Type = obj.TYPE_REG
 		p.From.Reg = v.Args[1].Reg()
 		p.To.Type = obj.TYPE_MEM
 		p.To.Reg = v.Args[0].Reg()
-		s.Prog(riscv.AFENCE)
+
+		p2 := s.Prog(riscv.AFENCE)
+		p2.From.Type = obj.TYPE_SPECIAL
+		p2.From.Offset = int64(riscv.SPOP_FENCE_RW)
+		p2.To.Type = obj.TYPE_SPECIAL
+		p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
 
 	case ssa.OpRISCV64LoweredAtomicStore32, ssa.OpRISCV64LoweredAtomicStore64:
 		as := riscv.AAMOSWAPW
@@ -961,8 +981,12 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 		p.To.Reg = v.Reg()
 
 	case ssa.OpRISCV64LoweredPubBarrier:
-		// FENCE
-		s.Prog(v.Op.Asm())
+		// FENCE W, W
+		p := s.Prog(v.Op.Asm())
+		p.From.Type = obj.TYPE_SPECIAL
+		p.From.Offset = int64(riscv.SPOP_FENCE_W)
+		p.To.Type = obj.TYPE_SPECIAL
+		p.To.Offset = int64(riscv.SPOP_FENCE_W)
 
 	case ssa.OpRISCV64LoweredRound32F, ssa.OpRISCV64LoweredRound64F:
 		// input is already rounded

src/internal/runtime/atomic/atomic_riscv64.s

@@ -82,9 +82,9 @@ TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
 // func Load8(ptr *uint8) uint8
 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
 	MOV	ptr+0(FP), A0
-	FENCE
+	FENCE	RW, RW
 	MOVBU	(A0), A1
-	FENCE
+	FENCE	R, RW
 	MOVB	A1, ret+8(FP)
 	RET
 
@@ -106,9 +106,9 @@ TEXT ·Store(SB), NOSPLIT, $0-12
 TEXT ·Store8(SB), NOSPLIT, $0-9
 	MOV	ptr+0(FP), A0
 	MOVBU	val+8(FP), A1
-	FENCE
+	FENCE	RW, W
 	MOVB	A1, (A0)
-	FENCE
+	FENCE	RW, RW
 	RET
 
 // func Store64(ptr *uint64, val uint64)

src/runtime/atomic_riscv64.s

@@ -6,5 +6,5 @@
 
 // func publicationBarrier()
 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
-	FENCE
+	FENCE	W, W
 	RET

src/runtime/sys_freebsd_riscv64.s

@@ -96,9 +96,9 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4
 TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8
 	MOV	wait+0(FP), A0
 	// We're done using the stack.
-	FENCE
+	FENCE	RW, W
 	MOVW	ZERO, (A0)
-	FENCE
+	FENCE	RW, RW
 	MOV	$0, A0	// exit code
 	MOV	$SYS_thr_exit, T0
 	ECALL

src/runtime/sys_linux_riscv64.s

@@ -60,9 +60,9 @@ TEXT runtime·exit<ABIInternal>(SB),NOSPLIT,$0
 TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8
 	MOV	wait+0(FP), A0
 	// We're done using the stack.
-	FENCE
+	FENCE	RW, W
 	MOVW	ZERO, (A0)
-	FENCE
+	FENCE	RW, RW
 	MOV	$0, A0	// exit code
 	MOV	$SYS_exit, A7
 	ECALL