runtime,syscall: implement and use syscalln on darwin

All darwin syscall implementations can be consolidated into a
single syscalln function, as already happens on Windows.

This reduces duplication and allows moving some logic from
runtime to syscall.

Updates #699135

Cq-Include-Trybots: luci.golang.try:gotip-darwin-arm64-longtest,gotip-darwin-amd64-longtest,x_sys-gotip-darwin-arm64-longtest,x_sys-gotip-darwin-amd64-longtest
Change-Id: If5de80442b1d4a1123258401a3ae21695e7c8f6b
Reviewed-on: https://go-review.googlesource.com/c/go/+/699177
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>

src/internal/trace/trace_test.go

@@ -448,6 +448,9 @@ func TestTraceStacks(t *testing.T) {
 					{"main.main.func11", 0},
 				}},
 			}...)
+			if runtime.GOOS == "darwin" {
+				want[len(want)-1].frames = append([]frame{{"syscall.syscall", 0}}, want[len(want)-1].frames...)
+			}
 		}
 		stackMatches := func(stk trace.Stack, frames []frame) bool {
 			for i, f := range slices.Collect(stk.Frames()) {

src/runtime/os_darwin.go

@@ -15,6 +15,11 @@ type mOS struct {
 	mutex       pthreadmutex
 	cond        pthreadcond
 	count       int
+
+	// address of errno variable for this thread.
+	// This is an optimization to avoid calling libc_error
+	// on every syscall_rawsyscalln.
+	errnoAddr *int32
 }
 
 func unimplemented(name string) {
@@ -330,6 +335,7 @@ func minit() {
 	}
 	minitSignalMask()
 	getg().m.procid = uint64(pthread_self())
+	libc_error_addr(&getg().m.errnoAddr)
 }
 
 // Called from dropm to undo the effect of an minit.

src/runtime/sys_darwin.go

@@ -10,186 +10,65 @@ import (
 	"unsafe"
 )
 
-//go:nosplit
-func libcError() uintptr {
-	errPtr, _ := syscall(abi.FuncPCABI0(libc_error_trampoline), 0, 0, 0)
-	return errPtr
-}
 func libc_error_trampoline()
 
-// The X versions of syscall expect the libc call to return a 64-bit result.
+// libc_error_addr puts the libc error
-// Otherwise (the non-X version) expects a 32-bit result.
+// address into addr.
-// This distinction is required because an error is indicated by returning -1,
-// and we need to know whether to check 32 or 64 bits of the result.
-// (Some libc functions that return 32 bits put junk in the upper 32 bits of AX.)
-
-//go:nosplit
-func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr) {
-	args := struct{ fn, a1, a2, a3, r1, r2 uintptr }{fn, a1, a2, a3, r1, r2}
-	libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall_trampoline)), unsafe.Pointer(&args))
-	return args.r1, args.r2
-}
-func syscall_trampoline()
-
-// golang.org/x/sys linknames syscall_syscall
-// (in addition to standard package syscall).
-// Do not remove or change the type signature.
 //
-//go:linkname syscall_syscall syscall.syscall
 //go:nosplit
-func syscall_syscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
+//go:cgo_unsafe_args
+func libc_error_addr(addr **int32) {
+	libcCall(unsafe.Pointer(abi.FuncPCABI0(libc_error_trampoline)), unsafe.Pointer(&addr))
+}
+
+// libcCallInfo is a structure used to pass parameters to the system call.
+type libcCallInfo struct {
+	fn     uintptr
+	n      uintptr // number of parameters
+	args   uintptr // parameters
+	r1, r2 uintptr // return values
+}
+
+// syscall_syscalln is a wrapper around the libc call with variable arguments.
+//
+//go:linkname syscall_syscalln syscall.syscalln
+//go:nosplit
+//go:uintptrkeepalive
+func syscall_syscalln(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
 	entersyscall()
-	r1, r2, err = syscall_rawSyscall(fn, a1, a2, a3)
+	r1, r2, err = syscall_rawsyscalln(fn, args...)
 	exitsyscall()
 	return r1, r2, err
 }
 
-//go:linkname syscall_syscallX syscall.syscallX
+// syscall_rawsyscalln is a wrapper around the libc call with variable arguments.
-//go:nosplit
+// The scheduler is not notified about the system call.
-func syscall_syscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
+// The syscall is executed on the current goroutine thread rather than on a
-	entersyscall()
+// dedicated syscall thread.
-	r1, r2, err = syscall_rawSyscallX(fn, a1, a2, a3)
-	exitsyscall()
-	return r1, r2, err
-}
-
-//go:nosplit
-func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr) {
-	args := struct{ fn, a1, a2, a3, a4, a5, a6, r1, r2 uintptr }{fn, a1, a2, a3, a4, a5, a6, r1, r2}
-	libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall6_trampoline)), unsafe.Pointer(&args))
-	return args.r1, args.r2
-}
-func syscall6_trampoline()
-
-// golang.org/x/sys linknames syscall.syscall6
-// (in addition to standard package syscall).
-// Do not remove or change the type signature.
 //
-// syscall.syscall6 is meant for package syscall (and x/sys),
+//go:linkname syscall_rawsyscalln syscall.rawsyscalln
-// but widely used packages access it using linkname.
-// Notable members of the hall of shame include:
-//   - github.com/tetratelabs/wazero
-//
-// See go.dev/issue/67401.
-//
-//go:linkname syscall_syscall6 syscall.syscall6
 //go:nosplit
-func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
+//go:uintptrkeepalive
-	entersyscall()
+func syscall_rawsyscalln(fn uintptr, args ...uintptr) (r1, r2, err uintptr) {
-	r1, r2, err = syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6)
+	c := &libcCallInfo{
-	exitsyscall()
+		fn: fn,
-	return r1, r2, err
+		n:  uintptr(len(args)),
-}
-
-//go:linkname syscall_syscall6X syscall.syscall6X
-//go:nosplit
-func syscall_syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
-	entersyscall()
-	r1, r2, err = syscall_rawSyscall6X(fn, a1, a2, a3, a4, a5, a6)
-	exitsyscall()
-	return r1, r2, err
-}
-
-//go:nosplit
-func syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr) {
-	args := struct{ fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, r1, r2 uintptr }{fn, a1, a2, a3, a4, a5, a6, a7, a8, a9, r1, r2}
-	libcCall(unsafe.Pointer(abi.FuncPCABI0(syscall9_trampoline)), unsafe.Pointer(&args))
-	return args.r1, args.r2
-}
-func syscall9_trampoline()
-
-// golang.org/x/sys linknames syscall.syscall9
-// (in addition to standard package syscall).
-// Do not remove or change the type signature.
-//
-//go:linkname syscall_syscall9 syscall.syscall9
-//go:nosplit
-func syscall_syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) {
-	entersyscall()
-	r1, r2, err = syscall_rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9)
-	exitsyscall()
-	return r1, r2, err
-}
-
-// golang.org/x/sys linknames syscall.syscallPtr
-// (in addition to standard package syscall).
-// Do not remove or change the type signature.
-//
-//go:linkname syscall_syscallPtr syscall.syscallPtr
-//go:nosplit
-func syscall_syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
-	entersyscall()
-	r1, r2, err = syscall_rawSyscallPtr(fn, a1, a2, a3)
-	exitsyscall()
-	return r1, r2, err
-}
-
-// golang.org/x/sys linknames syscall_rawSyscall
-// (in addition to standard package syscall).
-// Do not remove or change the type signature.
-//
-//go:linkname syscall_rawSyscall syscall.rawSyscall
-//go:nosplit
-func syscall_rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
-	r1, r2 = syscall(fn, a1, a2, a3)
-	// Check if r1 low 32 bits is -1, indicating an error.
-	if int32(r1) == -1 {
-		err = libcError()
 	}
-	return r1, r2, err
+	if c.n != 0 {
+		c.args = uintptr(noescape(unsafe.Pointer(&args[0])))
+	}
+	libcCall(unsafe.Pointer(abi.FuncPCABI0(syscallN_trampoline)), unsafe.Pointer(c))
+	if gp := getg(); gp != nil && gp.m != nil && gp.m.errnoAddr != nil {
+		err = uintptr(*gp.m.errnoAddr)
+	} else {
+		var errnoAddr *int32
+		libc_error_addr(&errnoAddr)
+		err = uintptr(*errnoAddr)
+	}
+	return c.r1, c.r2, err
 }
 
-//go:nosplit
+func syscallN_trampoline()
-func syscall_rawSyscallX(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
-	r1, r2 = syscall(fn, a1, a2, a3)
-	if r1 == ^uintptr(0) {
-		err = libcError()
-	}
-	return r1, r2, err
-}
-
-//go:nosplit
-func syscall_rawSyscallPtr(fn, a1, a2, a3 uintptr) (r1, r2, err uintptr) {
-	r1, r2 = syscall(fn, a1, a2, a3)
-	if r1 == 0 {
-		err = libcError()
-	}
-	return r1, r2, err
-}
-
-// golang.org/x/sys linknames syscall_rawSyscall6
-// (in addition to standard package syscall).
-// Do not remove or change the type signature.
-//
-//go:linkname syscall_rawSyscall6 syscall.rawSyscall6
-//go:nosplit
-func syscall_rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
-	r1, r2 = syscall6(fn, a1, a2, a3, a4, a5, a6)
-	// Check if r1 low 32 bits is -1, indicating an error.
-	if int32(r1) == -1 {
-		err = libcError()
-	}
-	return r1, r2, err
-}
-
-//go:nosplit
-func syscall_rawSyscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
-	r1, r2 = syscall6(fn, a1, a2, a3, a4, a5, a6)
-	if r1 == ^uintptr(0) {
-		err = libcError()
-	}
-	return r1, r2, err
-}
-
-//go:nosplit
-func syscall_rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) {
-	r1, r2 = syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9)
-	// Check if r1 low 32 bits is -1, indicating an error.
-	if int32(r1) == -1 {
-		err = libcError()
-	}
-	return r1, r2, err
-}
 
 // crypto_x509_syscall is used in crypto/x509/internal/macos to call into Security.framework and CF.
 

src/runtime/sys_darwin_amd64.s

@@ -506,118 +506,73 @@ TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0
 	CALL	libc_arc4random_buf(SB)
 	RET
 
-// syscall_trampoline calls a function in libc on behalf of the syscall package.
-// syscall_trampoline takes a pointer to a struct like:
-// struct {
-//	fn    uintptr
-//	a1    uintptr
-//	a2    uintptr
-//	a3    uintptr
-//	r1    uintptr
-//	r2    uintptr
-// }
-// syscall_trampoline must be called on the g0 stack with the
-// C calling convention (use libcCall).
-TEXT runtime·syscall_trampoline(SB),NOSPLIT,$16
-	MOVQ	(0*8)(DI), CX // fn
-	MOVQ	(2*8)(DI), SI // a2
-	MOVQ	(3*8)(DI), DX // a3
-	MOVQ	DI, (SP)
-	MOVQ	(1*8)(DI), DI // a1
-	XORL	AX, AX	      // vararg: say "no float args"
 
-	CALL	CX
+TEXT runtime·syscallN_trampoline(SB),NOSPLIT,$0
+	// store argument and original SP in a callee-saved register
+	MOVQ	DI, R13
+	MOVQ	SP, R14
 
-	MOVQ	(SP), DI
+	MOVQ	libcCallInfo_fn(R13), R11
-	MOVQ	AX, (4*8)(DI) // r1
+	MOVQ	libcCallInfo_n(R13), CX
-	MOVQ	DX, (5*8)(DI) // r2
+	MOVQ	libcCallInfo_args(R13), R10
 
-	XORL	AX, AX        // no error (it's ignored anyway)
+	// Fast version, do not store args on the stack.
-	RET
+	CMPL	CX, $0; JE _0args
+	CMPL	CX, $1; JE _1args
+	CMPL	CX, $2; JE _2args
+	CMPL	CX, $3; JE _3args
+	CMPL	CX, $4; JE _4args
+	CMPL	CX, $5; JE _5args
+	CMPL	CX, $6; JE _6args
 
-// syscall6_trampoline calls a function in libc on behalf of the syscall package.
+	// Reserve stack space for remaining args
-// syscall6_trampoline takes a pointer to a struct like:
+	MOVQ	CX, R12
-// struct {
+	SUBQ	$6, R12
-//	fn    uintptr
+	ADDQ	$1, R12	// make even number of words for stack alignment
-//	a1    uintptr
+	ANDQ	$~1, R12
-//	a2    uintptr
+	SHLQ	$3, R12
-//	a3    uintptr
+	SUBQ	R12, SP
-//	a4    uintptr
+
-//	a5    uintptr
+	// Copy args to the stack.
-//	a6    uintptr
+	// CX: count of stack arguments (n-6)
-//	r1    uintptr
+	// SI: &args[6]
-//	r2    uintptr
+	// DI: copy of RSP
-// }
+	SUBQ	$6, CX
-// syscall6_trampoline must be called on the g0 stack with the
+	MOVQ	R10, SI
-// C calling convention (use libcCall).
+	ADDQ	$(8*6), SI
-TEXT runtime·syscall6_trampoline(SB),NOSPLIT,$16
+	MOVQ	SP, DI
-	MOVQ	(0*8)(DI), R11// fn
+	CLD
-	MOVQ	(2*8)(DI), SI // a2
+	REP; MOVSQ
-	MOVQ	(3*8)(DI), DX // a3
+
-	MOVQ	(4*8)(DI), CX // a4
+_6args:
-	MOVQ	(5*8)(DI), R8 // a5
+	MOVQ	(5*8)(R10), R9
-	MOVQ	(6*8)(DI), R9 // a6
+_5args:
-	MOVQ	DI, (SP)
+	MOVQ	(4*8)(R10), R8
-	MOVQ	(1*8)(DI), DI // a1
+_4args:
-	XORL	AX, AX	      // vararg: say "no float args"
+	MOVQ	(3*8)(R10), CX
+_3args:
+	MOVQ	(2*8)(R10), DX
+_2args:
+	MOVQ	(1*8)(R10), SI
+_1args:
+	MOVQ	(0*8)(R10), DI
+_0args:
+
+	XORL	AX, AX	// vararg: say "no float args"
 
 	CALL	R11
 
-	MOVQ	(SP), DI
+	MOVQ	R14, SP	// free stack space
-	MOVQ	AX, (7*8)(DI) // r1
-	MOVQ	DX, (8*8)(DI) // r2
 
-	XORL	AX, AX        // no error (it's ignored anyway)
+	// Return result.
-	RET
+	MOVQ	AX, libcCallInfo_r1(R13)
-
+	MOVQ	DX, libcCallInfo_r2(R13)
-// syscall9_trampoline calls a function in libc on behalf of the syscall package.
-// syscall9_trampoline takes a pointer to a struct like:
-// struct {
-//	fn    uintptr
-//	a1    uintptr
-//	a2    uintptr
-//	a3    uintptr
-//	a4    uintptr
-//	a5    uintptr
-//	a6    uintptr
-//	a7    uintptr
-//	a8    uintptr
-//	a9    uintptr
-//	r1    uintptr
-//	r2    uintptr
-//	err   uintptr
-// }
-// syscall9_trampoline must be called on the g0 stack with the
-// C calling convention (use libcCall).
-TEXT runtime·syscall9_trampoline(SB),NOSPLIT,$32
-	MOVQ	(0*8)(DI), R13// fn
-	MOVQ	(2*8)(DI), SI // a2
-	MOVQ	(3*8)(DI), DX // a3
-	MOVQ	(4*8)(DI), CX // a4
-	MOVQ	(5*8)(DI), R8 // a5
-	MOVQ	(6*8)(DI), R9 // a6
-	MOVQ	(7*8)(DI), R10 // a7
-	MOVQ	R10, 0(SP)
-	MOVQ	(8*8)(DI), R11 // a8
-	MOVQ	R11, 8(SP)
-	MOVQ	(9*8)(DI), R12 // a9
-	MOVQ	R12, 16(SP)
-	MOVQ	DI, 24(SP)
-	MOVQ	(1*8)(DI), DI // a1
-	XORL	AX, AX	      // vararg: say "no float args"
-
-	CALL	R13
-
-	MOVQ	24(SP), DI
-	MOVQ	AX, (10*8)(DI) // r1
-	MOVQ	DX, (11*8)(DI) // r2
-
-	XORL	AX, AX        // no error (it's ignored anyway)
 	RET
 
 TEXT runtime·libc_error_trampoline(SB),NOSPLIT,$0
+	MOVQ	0(DI), R14
 	CALL	libc_error(SB)
-	MOVLQSX	(AX), AX
+	MOVQ	AX, (R14)
 	RET
 
 // syscall_x509 is for crypto/x509. It is like syscall6 but does not check for errors,

src/runtime/sys_darwin_arm64.s

@@ -481,128 +481,98 @@ TEXT runtime·arc4random_buf_trampoline(SB),NOSPLIT,$0
 	BL	libc_arc4random_buf(SB)
 	RET
 
-// syscall_trampoline calls a function in libc on behalf of the syscall package.
+TEXT runtime·syscallN_trampoline(SB),NOSPLIT,$16
-// syscall_trampoline takes a pointer to a struct like:
+	STP	(R19, R20), 16(RSP)	// save old R19, R20
-// struct {
+	MOVD	R0, R19	// save struct pointer
-//	fn    uintptr
+	MOVD	RSP, R20	// save stack pointer
-//	a1    uintptr
+	SUB	$16, RSP	// reserve 16 bytes for sp-8 where fp may be saved.
-//	a2    uintptr
-//	a3    uintptr
-//	r1    uintptr
-//	r2    uintptr
-// }
-// syscall_trampoline must be called on the g0 stack with the
-// C calling convention (use libcCall).
-TEXT runtime·syscall_trampoline(SB),NOSPLIT,$0
-	SUB	$16, RSP // push structure pointer
-	MOVD	R0, 8(RSP)
 
-	MOVD	0(R0), R12	// fn
+	MOVD	libcCallInfo_args(R19), R12
-	MOVD	16(R0), R1	// a2
+	// Do we have more than 8 arguments?
-	MOVD	24(R0), R2	// a3
+	MOVD	libcCallInfo_n(R19), R0
-	MOVD	8(R0), R0	// a1
+	CMP	$0, R0; BEQ _0args
+	CMP	$1, R0; BEQ _1args
+	CMP	$2, R0; BEQ _2args
+	CMP	$3, R0; BEQ _3args
+	CMP	$4, R0; BEQ _4args
+	CMP	$5, R0; BEQ _5args
+	CMP	$6, R0; BEQ _6args
+	CMP	$7, R0; BEQ _7args
+	CMP	$8, R0; BEQ _8args
+
+	// Reserve stack space for remaining args
+	SUB	$8, R0, R2
+	ADD	$1, R2, R3	// make even number of words for stack alignment
+	AND	$~1, R3
+	LSL	$3, R3
+	SUB	R3, RSP
+
+	// R4: size of stack arguments (n-8)*8
+	// R5: &args[8]
+	// R6: loop counter, from 0 to (n-8)*8
+	// R7: scratch
+	// R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR)
+	SUB	$8, R0, R4
+	LSL	$3, R4
+	ADD	$(8*8), R12, R5
+	MOVD	$0, R6
+	MOVD	RSP, R8
+stackargs:
+	MOVD	(R6)(R5), R7
+	MOVD	R7, (R6)(R8)
+	ADD	$8, R6
+	CMP	R6, R4
+	BNE	stackargs
+
+_8args:
+	MOVD	(7*8)(R12), R7
+_7args:
+	MOVD	(6*8)(R12), R6
+_6args:
+	MOVD	(5*8)(R12), R5
+_5args:
+	MOVD	(4*8)(R12), R4
+_4args:
+	MOVD	(3*8)(R12), R3
+_3args:
+	MOVD	(2*8)(R12), R2
+_2args:
+	MOVD	(1*8)(R12), R1
+_1args:
+	MOVD	(0*8)(R12), R0
+_0args:
 
 	// If fn is declared as vararg, we have to pass the vararg arguments on the stack.
 	// (Because ios decided not to adhere to the standard arm64 calling convention, sigh...)
-	// The only libSystem calls we support that are vararg are open, fcntl, and ioctl,
+	// The only libSystem calls we support with vararg are open, fcntl, ioctl,
-	// which are all of the form fn(x, y, ...). So we just need to put the 3rd arg
+	// which are all of the form fn(x, y, ...), and openat, which is of the form fn(x, y, z, ...).
-	// on the stack as well.
+	// So we just need to put the  3rd and the 4th arg on the stack as well.
+	// Note that historically openat has been called with syscall6, so we need to handle that case too.
 	// If we ever have other vararg libSystem calls, we might need to handle more cases.
+	MOVD	libcCallInfo_n(R19), R12
+	CMP	$3, R12; BNE 2(PC);
 	MOVD	R2, (RSP)
-
+	CMP $4, R12; BNE 2(PC);
-	BL	(R12)
+	MOVD	R3, (RSP)
-
+	CMP $6, R12; BNE 2(PC);
-	MOVD	8(RSP), R2	// pop structure pointer
-	ADD	$16, RSP
-	MOVD	R0, 32(R2)	// save r1
-	MOVD	R1, 40(R2)	// save r2
-	RET
-
-// syscall6_trampoline calls a function in libc on behalf of the syscall package.
-// syscall6_trampoline takes a pointer to a struct like:
-// struct {
-//	fn    uintptr
-//	a1    uintptr
-//	a2    uintptr
-//	a3    uintptr
-//	a4    uintptr
-//	a5    uintptr
-//	a6    uintptr
-//	r1    uintptr
-//	r2    uintptr
-// }
-// syscall6_trampoline must be called on the g0 stack with the
-// C calling convention (use libcCall).
-TEXT runtime·syscall6_trampoline(SB),NOSPLIT,$0
-	SUB	$16, RSP	// push structure pointer
-	MOVD	R0, 8(RSP)
-
-	MOVD	0(R0), R12	// fn
-	MOVD	16(R0), R1	// a2
-	MOVD	24(R0), R2	// a3
-	MOVD	32(R0), R3	// a4
-	MOVD	40(R0), R4	// a5
-	MOVD	48(R0), R5	// a6
-	MOVD	8(R0), R0	// a1
-
-	// If fn is declared as vararg, we have to pass the vararg arguments on the stack.
-	// See syscall_trampoline above. The only function this applies to is openat, for which the 4th
-	// arg must be on the stack.
 	MOVD	R3, (RSP)
 
+	MOVD	libcCallInfo_fn(R19), R12
 	BL	(R12)
 
-	MOVD	8(RSP), R2	// pop structure pointer
+	MOVD	R20, RSP	// free stack space
-	ADD	$16, RSP
-	MOVD	R0, 56(R2)	// save r1
-	MOVD	R1, 64(R2)	// save r2
-	RET
 
-// syscall9_trampoline calls a function in libc on behalf of the syscall package.
+	MOVD	R0, libcCallInfo_r1(R19)
-// syscall9_trampoline takes a pointer to a struct like:
+	MOVD	R1, libcCallInfo_r2(R19)
-// struct {
-//	fn    uintptr
-//	a1    uintptr
-//	a2    uintptr
-//	a3    uintptr
-//	a4    uintptr
-//	a5    uintptr
-//	a6    uintptr
-//	a7    uintptr
-//	a8    uintptr
-//	a9    uintptr
-//	r1    uintptr
-//	r2    uintptr
-// }
-// syscall9_trampoline must be called on the g0 stack with the
-// C calling convention (use libcCall).
-TEXT runtime·syscall9_trampoline(SB),NOSPLIT,$0
-	SUB	$16, RSP	// push structure pointer
-	MOVD	R0, 8(RSP)
 
-	MOVD	0(R0), R12	// fn
+	// Restore callee-saved registers.
-	MOVD	16(R0), R1	// a2
+	LDP	16(RSP), (R19, R20)
-	MOVD	24(R0), R2	// a3
+    RET
-	MOVD	32(R0), R3	// a4
-	MOVD	40(R0), R4	// a5
-	MOVD	48(R0), R5	// a6
-	MOVD	56(R0), R6	// a7
-	MOVD	64(R0), R7	// a8
-	MOVD	72(R0), R8	// a9
-	MOVD	R8, 0(RSP)	// the 9th arg and onwards must be passed on the stack
-	MOVD	8(R0), R0	// a1
-
-	BL	(R12)
-
-	MOVD	8(RSP), R2	// pop structure pointer
-	ADD	$16, RSP
-	MOVD	R0, 80(R2)	// save r1
-	MOVD	R1, 88(R2)	// save r2
-	RET
 
 TEXT runtime·libc_error_trampoline(SB),NOSPLIT,$0
+	MOVD	0(R0), R20
 	BL	libc_error(SB)
-	MOVW	(R0), R0
+	MOVD	R0, (R20)
 	RET
 
 // syscall_x509 is for crypto/x509. It is like syscall6 but does not check for errors,

src/syscall/syscall_darwin.go

@@ -323,10 +323,126 @@ func Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) {
 	return n, nil
 }
 
+// errno return e if int32(r) is -1, else it returns 0.
+//
+//go:nosplit
+func errno(r uintptr, e Errno) Errno {
+	if int32(r) == -1 {
+		return e
+	}
+	return 0
+}
+
+// errnoX return e if r is -1, else it returns 0.
+//
+//go:nosplit
+func errnoX(r uintptr, e Errno) Errno {
+	if r == ^uintptr(0) {
+		return e
+	}
+	return 0
+}
+
+// errnoPtr return e if r is 0, else it returns 0.
+//
+//go:nosplit
+func errnoPtr(r uintptr, e Errno) Errno {
+	if r == 0 {
+		return e
+	}
+	return 0
+}
+
+//go:cgo_import_dynamic libc_error __error "/usr/lib/libSystem.B.dylib"
+
+// golang.org/x/sys linknames the following syscalls.
+// Do not remove or change the type signature.
+
+//go:linkname syscall
+//go:nosplit
+//go:uintptrkeepalive
+func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = syscalln(fn, a1, a2, a3)
+	return r1, r2, errno(r1, err)
+}
+
+//go:linkname syscallX
+//go:nosplit
+//go:uintptrkeepalive
+func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = syscalln(fn, a1, a2, a3)
+	return r1, r2, errnoX(r1, err)
+}
+
+// syscall.syscall6 is meant for package syscall (and x/sys),
+// but widely used packages access it using linkname.
+// Notable members of the hall of shame include:
+//   - github.com/tetratelabs/wazero
+//
+// See go.dev/issue/67401.
+//
+//go:linkname syscall6
+//go:nosplit
+//go:uintptrkeepalive
+func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6)
+	return r1, r2, errno(r1, err)
+}
+
+//go:linkname syscall6X
+//go:nosplit
+//go:uintptrkeepalive
+func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6)
+	return r1, r2, errnoX(r1, err)
+}
+
+// syscall9 is used in [internal/syscall/unix].
+//
+//go:linkname syscall9
+//go:nosplit
+//go:uintptrkeepalive
+func syscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = syscalln(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9)
+	return r1, r2, errno(r1, err)
+}
+
+//go:linkname rawSyscall
+//go:nosplit
+//go:uintptrkeepalive
+func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = rawsyscalln(fn, a1, a2, a3)
+	return r1, r2, errno(r1, err)
+}
+
+//go:linkname rawSyscall6
+//go:nosplit
+//go:uintptrkeepalive
+func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = rawsyscalln(fn, a1, a2, a3, a4, a5, a6)
+	return r1, r2, errno(r1, err)
+}
+
+//go:linkname rawSyscall9
+//go:nosplit
+//go:uintptrkeepalive
+func rawSyscall9(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, err = rawsyscalln(fn, a1, a2, a3, a4, a5, a6, a7, a8, a9)
+	return r1, r2, errno(r1, err)
+}
+
+//go:linkname syscallPtr
+//go:nosplit
+//go:uintptrkeepalive
+func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno) {
+	r1, r2, e1 := syscalln(fn, a1, a2, a3)
+	return r1, r2, errnoPtr(r1, e1)
+}
+
 // Implemented in the runtime package (runtime/sys_darwin.go)
-func syscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
+
-func syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
+//go:noescape
-func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
+func syscalln(fn uintptr, args ...uintptr) (r1, r2 uintptr, err Errno)
-func rawSyscall(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
+
-func rawSyscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno)
+//go:noescape
-func syscallPtr(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
+func rawsyscalln(fn uintptr, args ...uintptr) (r1, r2 uintptr, err Errno)

src/syscall/syscall_darwin_amd64.go

@@ -61,7 +61,4 @@ func libc_sendfile_trampoline()
 
 //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib"
 
-// Implemented in the runtime package (runtime/sys_darwin_64.go)
-func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
-
 func Syscall9(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno)

src/syscall/syscall_darwin_arm64.go

@@ -61,7 +61,4 @@ func libc_sendfile_trampoline()
 
 //go:cgo_import_dynamic libc_sendfile sendfile "/usr/lib/libSystem.B.dylib"
 
-// Implemented in the runtime package (runtime/sys_darwin_64.go)
-func syscallX(fn, a1, a2, a3 uintptr) (r1, r2 uintptr, err Errno)
-
 func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err Errno) // sic