diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go index 76cf59772bc..dcdfe666aca 100644 --- a/src/runtime/os3_solaris.go +++ b/src/runtime/os3_solaris.go @@ -7,6 +7,7 @@ package runtime import ( "internal/abi" "internal/goarch" + "runtime/internal/atomic" "unsafe" ) @@ -182,7 +183,7 @@ func newosproc(mp *m) { } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Solaris because we let // libc clean up threads. throw("exitThread") diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go index 15e4929779a..104c397e8c0 100644 --- a/src/runtime/os_aix.go +++ b/src/runtime/os_aix.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -233,7 +234,7 @@ func newosproc(mp *m) { } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on AIX because we let // libc clean up threads. throw("exitThread") diff --git a/src/runtime/os_js.go b/src/runtime/os_js.go index 7ae0e8d3ece..7481fb92bf8 100644 --- a/src/runtime/os_js.go +++ b/src/runtime/os_js.go @@ -7,6 +7,7 @@ package runtime import ( + "runtime/internal/atomic" "unsafe" ) @@ -35,7 +36,7 @@ func usleep_no_g(usec uint32) { usleep(usec) } -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) type mOS struct{} diff --git a/src/runtime/os_openbsd_syscall2.go b/src/runtime/os_openbsd_syscall2.go index e4c9d2fe893..ab6b1818285 100644 --- a/src/runtime/os_openbsd_syscall2.go +++ b/src/runtime/os_openbsd_syscall2.go @@ -7,6 +7,7 @@ package runtime import ( + "runtime/internal/atomic" "unsafe" ) @@ -49,11 +50,11 @@ func open(name *byte, mode, perm int32) int32 // return value is only set on linux to be used in osinit() func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 -// exitThread terminates the current thread, writing *wait = 0 when +// exitThread terminates the current thread, writing *wait = freeMStack when // the stack is safe to reclaim. // //go:noescape -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) //go:noescape func obsdsigprocmask(how int32, new sigset) sigset diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index 2836ad9c67f..5e5a63dcbf9 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -468,7 +468,7 @@ func newosproc(mp *m) { } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Plan 9 because we let // the OS clean up threads. throw("exitThread") diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 54261d6fc02..44718f1d214 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -941,7 +941,7 @@ func newosproc0(mp *m, stk unsafe.Pointer) { throw("bad newosproc0") } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Windows because we let // the OS clean up threads. throw("exitThread") diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 02390375b5a..4285ff6b7c4 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1582,19 +1582,18 @@ func mexit(osStack bool) { } throw("m not found in allm") found: - if !osStack { - // Delay reaping m until it's done with the stack. - // - // If this is using an OS stack, the OS will free it - // so there's no need for reaping. - atomic.Store(&mp.freeWait, 1) - // Put m on the free list, though it will not be reaped until - // freeWait is 0. Note that the free list must not be linked - // through alllink because some functions walk allm without - // locking, so may be using alllink. - mp.freelink = sched.freem - sched.freem = mp - } + // Delay reaping m until it's done with the stack. + // + // Put mp on the free list, though it will not be reaped while freeWait + // is freeMWait. mp is no longer reachable via allm, so even if it is + // on an OS stack, we must keep a reference to mp alive so that the GC + // doesn't free mp while we are still using it. + // + // Note that the free list must not be linked through alllink because + // some functions walk allm without locking, so may be using alllink. + mp.freeWait.Store(freeMWait) + mp.freelink = sched.freem + sched.freem = mp unlock(&sched.lock) atomic.Xadd64(&ncgocall, int64(mp.ncgocall)) @@ -1624,6 +1623,9 @@ found: mdestroy(mp) if osStack { + // No more uses of mp, so it is safe to drop the reference. + mp.freeWait.Store(freeMRef) + // Return from mstart and let the system thread // library free the g0 stack and terminate the thread. return @@ -1795,19 +1797,25 @@ func allocm(pp *p, fn func(), id int64) *m { lock(&sched.lock) var newList *m for freem := sched.freem; freem != nil; { - if freem.freeWait != 0 { + wait := freem.freeWait.Load() + if wait == freeMWait { next := freem.freelink freem.freelink = newList newList = freem freem = next continue } - // stackfree must be on the system stack, but allocm is - // reachable off the system stack transitively from - // startm. - systemstack(func() { - stackfree(freem.g0.stack) - }) + // Free the stack if needed. For freeMRef, there is + // nothing to do except drop freem from the sched.freem + // list. + if wait == freeMStack { + // stackfree must be on the system stack, but allocm is + // reachable off the system stack transitively from + // startm. + systemstack(func() { + stackfree(freem.g0.stack) + }) + } freem = freem.freelink } sched.freem = newList diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 5b55b55ce1c..0392f2968fe 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -516,6 +516,13 @@ const ( tlsSize = tlsSlots * goarch.PtrSize ) +// Values for m.freeWait. +const ( + freeMStack = 0 // M done, free stack and reference. + freeMRef = 1 // M done, free reference. + freeMWait = 2 // M still in use. +) + type m struct { g0 *g // goroutine with scheduling stack morebuf gobuf // gobuf arg to morestack @@ -547,7 +554,7 @@ type m struct { printlock int8 incgo bool // m is executing a cgo call isextra bool // m is an extra m - freeWait uint32 // if == 0, safe to free g0 and delete m (atomic) + freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) fastrand uint64 needextram bool traceback uint8 diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index 94a888dec62..d2ad8d4ec88 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -6,7 +6,10 @@ package runtime -import "unsafe" +import ( + "runtime/internal/atomic" + "unsafe" +) // read calls the read system call. // It returns a non-negative number of bytes written or a negative errno value. @@ -34,8 +37,8 @@ func open(name *byte, mode, perm int32) int32 // return value is only set on linux to be used in osinit() func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 -// exitThread terminates the current thread, writing *wait = 0 when +// exitThread terminates the current thread, writing *wait = freeMStack when // the stack is safe to reclaim. // //go:noescape -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 1547fdceb0c..88af8944097 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -6,6 +6,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -474,7 +475,7 @@ func pthread_cond_signal(c *pthreadcond) int32 { func pthread_cond_signal_trampoline() // Not used on Darwin, but must be defined. -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { } //go:nosplit diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index 602d5e9b76b..0cf98219fb0 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -65,7 +65,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s index f919c5a000a..4e0bc9b08c5 100644 --- a/src/runtime/sys_freebsd_386.s +++ b/src/runtime/sys_freebsd_386.s @@ -99,7 +99,7 @@ GLOBL exitStack<>(SB),RODATA,$8 DATA exitStack<>+0x00(SB)/4, $0 DATA exitStack<>+0x04(SB)/4, $0 -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_amd64.s b/src/runtime/sys_freebsd_amd64.s index c266d73ea28..374e0ab769f 100644 --- a/src/runtime/sys_freebsd_amd64.s +++ b/src/runtime/sys_freebsd_amd64.s @@ -96,7 +96,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_arm.s b/src/runtime/sys_freebsd_arm.s index 89a8d2bfac2..a3fee1426c4 100644 --- a/src/runtime/sys_freebsd_arm.s +++ b/src/runtime/sys_freebsd_arm.s @@ -85,7 +85,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVW.CS R8, (R8) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_freebsd_arm64.s b/src/runtime/sys_freebsd_arm64.s index 4f24da62ef1..29866cb4ca4 100644 --- a/src/runtime/sys_freebsd_arm64.s +++ b/src/runtime/sys_freebsd_arm64.s @@ -99,7 +99,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 MOVD $0, R0 MOVD R0, (R0) -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_freebsd_riscv64.s b/src/runtime/sys_freebsd_riscv64.s index 3c1b966348d..4b8699e2bff 100644 --- a/src/runtime/sys_freebsd_riscv64.s +++ b/src/runtime/sys_freebsd_riscv64.s @@ -96,7 +96,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 ECALL WORD $0 // crash -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOV wait+0(FP), A0 // We're done using the stack. diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 4f5b34b9963..12a294153d2 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -72,7 +72,7 @@ TEXT exit1<>(SB),NOSPLIT,$0 INT $3 // not reached RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index d91f9bd31f1..c7a89ba5363 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -54,7 +54,7 @@ TEXT runtime·exit(SB),NOSPLIT,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index 1bc9e86d6d3..7b8c4f0e047 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -117,7 +117,7 @@ TEXT exit1<>(SB),NOSPLIT|NOFRAME,$0 MOVW $1003, R1 MOVW R0, (R1) // fail hard -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 04a2cd2da1f..38ff6ac3301 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -56,7 +56,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SVC RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_loong64.s b/src/runtime/sys_linux_loong64.s index 5705c374965..9ce5e722565 100644 --- a/src/runtime/sys_linux_loong64.s +++ b/src/runtime/sys_linux_loong64.s @@ -49,7 +49,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVV wait+0(FP), R19 // We're done using the stack. diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 5d113395849..47f2da524d1 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -51,7 +51,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVV wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index c4507c60988..5e6b6c15041 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -50,7 +50,7 @@ TEXT runtime·exit(SB),NOSPLIT,$0-4 UNDEF RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index 853008d5fe9..49974c6bf61 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -49,7 +49,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL $SYS_exit_group RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index f4fb1c124b2..d1558fd6f79 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -57,7 +57,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 ECALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOV wait+0(FP), A0 // We're done using the stack. diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index 777a4747d48..1448670b913 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -46,7 +46,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s index 581b4fc9b68..7be18c61d8b 100644 --- a/src/runtime/sys_netbsd_386.s +++ b/src/runtime/sys_netbsd_386.s @@ -53,7 +53,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-4 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index ab11f6ff669..30f3f380b6f 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -122,7 +122,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_netbsd_arm.s b/src/runtime/sys_netbsd_arm.s index dbe3dbcffc0..62fa852addd 100644 --- a/src/runtime/sys_netbsd_arm.s +++ b/src/runtime/sys_netbsd_arm.s @@ -56,7 +56,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVW.CS R8, (R8) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_netbsd_arm64.s b/src/runtime/sys_netbsd_arm64.s index fc126cad7d2..d57959f8d76 100644 --- a/src/runtime/sys_netbsd_arm64.s +++ b/src/runtime/sys_netbsd_arm64.s @@ -115,7 +115,7 @@ TEXT runtime·exit(SB),NOSPLIT,$-8 MOVD $0, R0 // If we're still running, MOVD R0, (R0) // crash -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go index f936e0cfc3b..f755cd528c8 100644 --- a/src/runtime/sys_openbsd2.go +++ b/src/runtime/sys_openbsd2.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -248,7 +249,7 @@ func sigaltstack(new *stackt, old *stackt) { func sigaltstack_trampoline() // Not used on OpenBSD, but must be defined. -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { } //go:nosplit diff --git a/src/runtime/sys_openbsd_mips64.s b/src/runtime/sys_openbsd_mips64.s index c2b20920531..cc37e52e161 100644 --- a/src/runtime/sys_openbsd_mips64.s +++ b/src/runtime/sys_openbsd_mips64.s @@ -24,7 +24,7 @@ TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVV R2, (R2) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0 MOVV wait+0(FP), R4 // arg 1 - notdead MOVV $302, R2 // sys___threxit