mirror of
https://github.com/golang/go.git
synced 2025-12-08 06:10:04 +00:00
runtime: improve Linux mutex
The implementation is hybrid active/passive spin/blocking mutex. The design minimizes amount of context switches and futex calls. The idea is that all critical sections in runtime are intentially small, so pure blocking mutex behaves badly causing a lot of context switches, thread parking/unparking and kernel calls. Note that some synthetic benchmarks become somewhat slower, that's due to increased contention on other data structures, it should not affect programs that do any real work. On 2 x Intel E5620, 8 HT cores, 2.4GHz benchmark old ns/op new ns/op delta BenchmarkSelectContended 521.00 503.00 -3.45% BenchmarkSelectContended-2 661.00 320.00 -51.59% BenchmarkSelectContended-4 1139.00 629.00 -44.78% BenchmarkSelectContended-8 2870.00 878.00 -69.41% BenchmarkSelectContended-16 5276.00 818.00 -84.50% BenchmarkChanContended 112.00 103.00 -8.04% BenchmarkChanContended-2 631.00 174.00 -72.42% BenchmarkChanContended-4 682.00 272.00 -60.12% BenchmarkChanContended-8 1601.00 520.00 -67.52% BenchmarkChanContended-16 3100.00 372.00 -88.00% BenchmarkChanSync 253.00 239.00 -5.53% BenchmarkChanSync-2 5030.00 4648.00 -7.59% BenchmarkChanSync-4 4826.00 4694.00 -2.74% BenchmarkChanSync-8 4778.00 4713.00 -1.36% BenchmarkChanSync-16 5289.00 4710.00 -10.95% BenchmarkChanProdCons0 273.00 254.00 -6.96% BenchmarkChanProdCons0-2 599.00 400.00 -33.22% BenchmarkChanProdCons0-4 1168.00 659.00 -43.58% BenchmarkChanProdCons0-8 2831.00 1057.00 -62.66% BenchmarkChanProdCons0-16 4197.00 1037.00 -75.29% BenchmarkChanProdCons10 150.00 140.00 -6.67% BenchmarkChanProdCons10-2 607.00 268.00 -55.85% BenchmarkChanProdCons10-4 1137.00 404.00 -64.47% BenchmarkChanProdCons10-8 2115.00 828.00 -60.85% BenchmarkChanProdCons10-16 4283.00 855.00 -80.04% BenchmarkChanProdCons100 117.00 110.00 -5.98% BenchmarkChanProdCons100-2 558.00 218.00 -60.93% BenchmarkChanProdCons100-4 722.00 287.00 -60.25% BenchmarkChanProdCons100-8 1840.00 431.00 -76.58% BenchmarkChanProdCons100-16 3394.00 448.00 -86.80% BenchmarkChanProdConsWork0 2014.00 1996.00 -0.89% BenchmarkChanProdConsWork0-2 1207.00 1127.00 -6.63% BenchmarkChanProdConsWork0-4 1913.00 611.00 -68.06% BenchmarkChanProdConsWork0-8 3016.00 949.00 -68.53% BenchmarkChanProdConsWork0-16 4320.00 1154.00 -73.29% BenchmarkChanProdConsWork10 1906.00 1897.00 -0.47% BenchmarkChanProdConsWork10-2 1123.00 1033.00 -8.01% BenchmarkChanProdConsWork10-4 1076.00 571.00 -46.93% BenchmarkChanProdConsWork10-8 2748.00 1096.00 -60.12% BenchmarkChanProdConsWork10-16 4600.00 1105.00 -75.98% BenchmarkChanProdConsWork100 1884.00 1852.00 -1.70% BenchmarkChanProdConsWork100-2 1235.00 1146.00 -7.21% BenchmarkChanProdConsWork100-4 1217.00 619.00 -49.14% BenchmarkChanProdConsWork100-8 1534.00 509.00 -66.82% BenchmarkChanProdConsWork100-16 4126.00 918.00 -77.75% BenchmarkSyscall 34.40 33.30 -3.20% BenchmarkSyscall-2 160.00 121.00 -24.38% BenchmarkSyscall-4 131.00 136.00 +3.82% BenchmarkSyscall-8 139.00 131.00 -5.76% BenchmarkSyscall-16 161.00 168.00 +4.35% BenchmarkSyscallWork 950.00 950.00 +0.00% BenchmarkSyscallWork-2 481.00 480.00 -0.21% BenchmarkSyscallWork-4 268.00 270.00 +0.75% BenchmarkSyscallWork-8 156.00 169.00 +8.33% BenchmarkSyscallWork-16 188.00 184.00 -2.13% BenchmarkSemaSyntNonblock 36.40 35.60 -2.20% BenchmarkSemaSyntNonblock-2 81.40 45.10 -44.59% BenchmarkSemaSyntNonblock-4 126.00 108.00 -14.29% BenchmarkSemaSyntNonblock-8 112.00 112.00 +0.00% BenchmarkSemaSyntNonblock-16 110.00 112.00 +1.82% BenchmarkSemaSyntBlock 35.30 35.30 +0.00% BenchmarkSemaSyntBlock-2 118.00 124.00 +5.08% BenchmarkSemaSyntBlock-4 105.00 108.00 +2.86% BenchmarkSemaSyntBlock-8 101.00 111.00 +9.90% BenchmarkSemaSyntBlock-16 112.00 118.00 +5.36% BenchmarkSemaWorkNonblock 810.00 811.00 +0.12% BenchmarkSemaWorkNonblock-2 476.00 414.00 -13.03% BenchmarkSemaWorkNonblock-4 238.00 228.00 -4.20% BenchmarkSemaWorkNonblock-8 140.00 126.00 -10.00% BenchmarkSemaWorkNonblock-16 117.00 116.00 -0.85% BenchmarkSemaWorkBlock 810.00 811.00 +0.12% BenchmarkSemaWorkBlock-2 454.00 466.00 +2.64% BenchmarkSemaWorkBlock-4 243.00 241.00 -0.82% BenchmarkSemaWorkBlock-8 145.00 137.00 -5.52% BenchmarkSemaWorkBlock-16 132.00 123.00 -6.82% BenchmarkContendedSemaphore 123.00 102.00 -17.07% BenchmarkContendedSemaphore-2 34.80 34.90 +0.29% BenchmarkContendedSemaphore-4 34.70 34.80 +0.29% BenchmarkContendedSemaphore-8 34.70 34.70 +0.00% BenchmarkContendedSemaphore-16 34.80 34.70 -0.29% BenchmarkMutex 26.80 26.00 -2.99% BenchmarkMutex-2 108.00 45.20 -58.15% BenchmarkMutex-4 103.00 127.00 +23.30% BenchmarkMutex-8 109.00 147.00 +34.86% BenchmarkMutex-16 102.00 152.00 +49.02% BenchmarkMutexSlack 27.00 26.90 -0.37% BenchmarkMutexSlack-2 149.00 165.00 +10.74% BenchmarkMutexSlack-4 121.00 209.00 +72.73% BenchmarkMutexSlack-8 101.00 158.00 +56.44% BenchmarkMutexSlack-16 97.00 129.00 +32.99% BenchmarkMutexWork 792.00 794.00 +0.25% BenchmarkMutexWork-2 407.00 409.00 +0.49% BenchmarkMutexWork-4 220.00 209.00 -5.00% BenchmarkMutexWork-8 267.00 160.00 -40.07% BenchmarkMutexWork-16 315.00 300.00 -4.76% BenchmarkMutexWorkSlack 792.00 793.00 +0.13% BenchmarkMutexWorkSlack-2 406.00 404.00 -0.49% BenchmarkMutexWorkSlack-4 225.00 212.00 -5.78% BenchmarkMutexWorkSlack-8 268.00 136.00 -49.25% BenchmarkMutexWorkSlack-16 300.00 300.00 +0.00% BenchmarkRWMutexWrite100 27.10 27.00 -0.37% BenchmarkRWMutexWrite100-2 33.10 40.80 +23.26% BenchmarkRWMutexWrite100-4 113.00 88.10 -22.04% BenchmarkRWMutexWrite100-8 119.00 95.30 -19.92% BenchmarkRWMutexWrite100-16 148.00 109.00 -26.35% BenchmarkRWMutexWrite10 29.60 29.40 -0.68% BenchmarkRWMutexWrite10-2 111.00 61.40 -44.68% BenchmarkRWMutexWrite10-4 270.00 208.00 -22.96% BenchmarkRWMutexWrite10-8 204.00 185.00 -9.31% BenchmarkRWMutexWrite10-16 261.00 190.00 -27.20% BenchmarkRWMutexWorkWrite100 1040.00 1036.00 -0.38% BenchmarkRWMutexWorkWrite100-2 593.00 580.00 -2.19% BenchmarkRWMutexWorkWrite100-4 470.00 365.00 -22.34% BenchmarkRWMutexWorkWrite100-8 468.00 289.00 -38.25% BenchmarkRWMutexWorkWrite100-16 604.00 374.00 -38.08% BenchmarkRWMutexWorkWrite10 951.00 951.00 +0.00% BenchmarkRWMutexWorkWrite10-2 1001.00 928.00 -7.29% BenchmarkRWMutexWorkWrite10-4 1555.00 1006.00 -35.31% BenchmarkRWMutexWorkWrite10-8 2085.00 1171.00 -43.84% BenchmarkRWMutexWorkWrite10-16 2082.00 1614.00 -22.48% R=rsc, iant, msolo, fw, iant CC=golang-dev https://golang.org/cl/4711045
This commit is contained in:
Dmitriy Vyukov
Russ Cox
parent
bed7e3ed78
commit
4e5086b993
18 changed files with 278 additions and 91 deletions
|
|
@ -8,6 +8,11 @@
|
|||
#include "stack.h"
|
||||
|
||||
extern SigTab runtime·sigtab[];
|
||||
static int32 proccount;
|
||||
|
||||
int32 runtime·open(uint8*, int32, int32);
|
||||
int32 runtime·close(int32);
|
||||
int32 runtime·read(int32, void*, int32);
|
||||
|
||||
// Linux futex.
|
||||
//
|
||||
|
|
@ -15,11 +20,19 @@ extern SigTab runtime·sigtab[];
|
|||
// futexwakeup(uint32 *addr)
|
||||
//
|
||||
// Futexsleep atomically checks if *addr == val and if so, sleeps on addr.
|
||||
// Futexwakeup wakes up one thread sleeping on addr.
|
||||
// Futexwakeup wakes up threads sleeping on addr.
|
||||
// Futexsleep is allowed to wake up spuriously.
|
||||
|
||||
enum
|
||||
{
|
||||
MUTEX_UNLOCKED = 0,
|
||||
MUTEX_LOCKED = 1,
|
||||
MUTEX_SLEEPING = 2,
|
||||
|
||||
ACTIVE_SPIN = 4,
|
||||
ACTIVE_SPIN_CNT = 30,
|
||||
PASSIVE_SPIN = 1,
|
||||
|
||||
FUTEX_WAIT = 0,
|
||||
FUTEX_WAKE = 1,
|
||||
|
||||
|
|
@ -52,13 +65,13 @@ futexsleep(uint32 *addr, uint32 val)
|
|||
runtime·futex(addr, FUTEX_WAIT, val, &longtime, nil, 0);
|
||||
}
|
||||
|
||||
// If any procs are sleeping on addr, wake up at least one.
|
||||
// If any procs are sleeping on addr, wake up at most cnt.
|
||||
static void
|
||||
futexwakeup(uint32 *addr)
|
||||
futexwakeup(uint32 *addr, uint32 cnt)
|
||||
{
|
||||
int64 ret;
|
||||
|
||||
ret = runtime·futex(addr, FUTEX_WAKE, 1, nil, nil, 0);
|
||||
ret = runtime·futex(addr, FUTEX_WAKE, cnt, nil, nil, 0);
|
||||
|
||||
if(ret >= 0)
|
||||
return;
|
||||
|
|
@ -66,70 +79,96 @@ futexwakeup(uint32 *addr)
|
|||
// I don't know that futex wakeup can return
|
||||
// EAGAIN or EINTR, but if it does, it would be
|
||||
// safe to loop and call futex again.
|
||||
|
||||
runtime·prints("futexwakeup addr=");
|
||||
runtime·printpointer(addr);
|
||||
runtime·prints(" returned ");
|
||||
runtime·printint(ret);
|
||||
runtime·prints("\n");
|
||||
runtime·printf("futexwakeup addr=%p returned %D\n", addr, ret);
|
||||
*(int32*)0x1006 = 0x1006;
|
||||
}
|
||||
|
||||
static int32
|
||||
getproccount(void)
|
||||
{
|
||||
int32 fd, rd, cnt, cpustrlen;
|
||||
byte *cpustr, *pos, *bufpos;
|
||||
byte buf[256];
|
||||
|
||||
// Lock and unlock.
|
||||
//
|
||||
// The lock state is a single 32-bit word that holds
|
||||
// a 31-bit count of threads waiting for the lock
|
||||
// and a single bit (the low bit) saying whether the lock is held.
|
||||
// The uncontended case runs entirely in user space.
|
||||
// When contention is detected, we defer to the kernel (futex).
|
||||
//
|
||||
// A reminder: compare-and-swap runtime·cas(addr, old, new) does
|
||||
// if(*addr == old) { *addr = new; return 1; }
|
||||
// else return 0;
|
||||
// but atomically.
|
||||
fd = runtime·open((byte*)"/proc/stat", O_RDONLY|O_CLOEXEC, 0);
|
||||
if(fd == -1)
|
||||
return 1;
|
||||
cnt = 0;
|
||||
bufpos = buf;
|
||||
cpustr = (byte*)"\ncpu";
|
||||
cpustrlen = runtime·findnull(cpustr);
|
||||
for(;;) {
|
||||
rd = runtime·read(fd, bufpos, sizeof(buf)-cpustrlen);
|
||||
if(rd == -1)
|
||||
break;
|
||||
bufpos[rd] = 0;
|
||||
for(pos=buf; pos=runtime·strstr(pos, cpustr); cnt++, pos++) {
|
||||
}
|
||||
if(rd < cpustrlen)
|
||||
break;
|
||||
runtime·memmove(buf, bufpos+rd-cpustrlen+1, cpustrlen-1);
|
||||
bufpos = buf+cpustrlen-1;
|
||||
}
|
||||
runtime·close(fd);
|
||||
return cnt ? cnt : 1;
|
||||
}
|
||||
|
||||
// Possible lock states are MUTEX_UNLOCKED, MUTEX_LOCKED and MUTEX_SLEEPING.
|
||||
// MUTEX_SLEEPING means that there is presumably at least one sleeping thread.
|
||||
// Note that there can be spinning threads during all states - they do not
|
||||
// affect mutex's state.
|
||||
static void
|
||||
futexlock(Lock *l)
|
||||
{
|
||||
uint32 v;
|
||||
uint32 i, v, wait, spin;
|
||||
|
||||
again:
|
||||
v = l->key;
|
||||
if((v&1) == 0){
|
||||
if(runtime·cas(&l->key, v, v|1)){
|
||||
// Lock wasn't held; we grabbed it.
|
||||
return;
|
||||
// Speculative grab for lock.
|
||||
v = runtime·xchg(&l->key, MUTEX_LOCKED);
|
||||
if(v == MUTEX_UNLOCKED)
|
||||
return;
|
||||
|
||||
// wait is either MUTEX_LOCKED or MUTEX_SLEEPING
|
||||
// depending on whether there is a thread sleeping
|
||||
// on this mutex. If we ever change l->key from
|
||||
// MUTEX_SLEEPING to some other value, we must be
|
||||
// careful to change it back to MUTEX_SLEEPING before
|
||||
// returning, to ensure that the sleeping thread gets
|
||||
// its wakeup call.
|
||||
wait = v;
|
||||
|
||||
if(proccount == 0)
|
||||
proccount = getproccount();
|
||||
|
||||
// On uniprocessor's, no point spinning.
|
||||
// On multiprocessors, spin for ACTIVE_SPIN attempts.
|
||||
spin = 0;
|
||||
if(proccount > 1)
|
||||
spin = ACTIVE_SPIN;
|
||||
|
||||
for(;;) {
|
||||
// Try for lock, spinning.
|
||||
for(i = 0; i < spin; i++) {
|
||||
while(l->key == MUTEX_UNLOCKED)
|
||||
if(runtime·cas(&l->key, MUTEX_UNLOCKED, wait))
|
||||
return;
|
||||
runtime·procyield(ACTIVE_SPIN_CNT);
|
||||
}
|
||||
goto again;
|
||||
|
||||
// Try for lock, rescheduling.
|
||||
for(i=0; i < PASSIVE_SPIN; i++) {
|
||||
while(l->key == MUTEX_UNLOCKED)
|
||||
if(runtime·cas(&l->key, MUTEX_UNLOCKED, wait))
|
||||
return;
|
||||
runtime·osyield();
|
||||
}
|
||||
|
||||
// Sleep.
|
||||
v = runtime·xchg(&l->key, MUTEX_SLEEPING);
|
||||
if(v == MUTEX_UNLOCKED)
|
||||
return;
|
||||
wait = MUTEX_SLEEPING;
|
||||
futexsleep(&l->key, MUTEX_SLEEPING);
|
||||
}
|
||||
|
||||
// Lock was held; try to add ourselves to the waiter count.
|
||||
if(!runtime·cas(&l->key, v, v+2))
|
||||
goto again;
|
||||
|
||||
// We're accounted for, now sleep in the kernel.
|
||||
//
|
||||
// We avoid the obvious lock/unlock race because
|
||||
// the kernel won't put us to sleep if l->key has
|
||||
// changed underfoot and is no longer v+2.
|
||||
//
|
||||
// We only really care that (v&1) == 1 (the lock is held),
|
||||
// and in fact there is a futex variant that could
|
||||
// accommodate that check, but let's not get carried away.)
|
||||
futexsleep(&l->key, v+2);
|
||||
|
||||
// We're awake: remove ourselves from the count.
|
||||
for(;;){
|
||||
v = l->key;
|
||||
if(v < 2)
|
||||
runtime·throw("bad lock key");
|
||||
if(runtime·cas(&l->key, v, v-2))
|
||||
break;
|
||||
}
|
||||
|
||||
// Try for the lock again.
|
||||
goto again;
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -137,34 +176,26 @@ futexunlock(Lock *l)
|
|||
{
|
||||
uint32 v;
|
||||
|
||||
// Atomically get value and clear lock bit.
|
||||
again:
|
||||
v = l->key;
|
||||
if((v&1) == 0)
|
||||
v = runtime·xchg(&l->key, MUTEX_UNLOCKED);
|
||||
if(v == MUTEX_UNLOCKED)
|
||||
runtime·throw("unlock of unlocked lock");
|
||||
if(!runtime·cas(&l->key, v, v&~1))
|
||||
goto again;
|
||||
|
||||
// If there were waiters, wake one.
|
||||
if(v & ~1)
|
||||
futexwakeup(&l->key);
|
||||
if(v == MUTEX_SLEEPING)
|
||||
futexwakeup(&l->key, 1);
|
||||
}
|
||||
|
||||
void
|
||||
runtime·lock(Lock *l)
|
||||
{
|
||||
if(m->locks < 0)
|
||||
runtime·throw("lock count");
|
||||
m->locks++;
|
||||
if(m->locks++ < 0)
|
||||
runtime·throw("runtime·lock: lock count");
|
||||
futexlock(l);
|
||||
}
|
||||
|
||||
void
|
||||
runtime·unlock(Lock *l)
|
||||
{
|
||||
m->locks--;
|
||||
if(m->locks < 0)
|
||||
runtime·throw("lock count");
|
||||
if(--m->locks < 0)
|
||||
runtime·throw("runtime·unlock: lock count");
|
||||
futexunlock(l);
|
||||
}
|
||||
|
||||
|
|
@ -175,35 +206,24 @@ runtime·destroylock(Lock*)
|
|||
|
||||
|
||||
// One-time notifications.
|
||||
//
|
||||
// Since the lock/unlock implementation already
|
||||
// takes care of sleeping in the kernel, we just reuse it.
|
||||
// (But it's a weird use, so it gets its own interface.)
|
||||
//
|
||||
// We use a lock to represent the event:
|
||||
// unlocked == event has happened.
|
||||
// Thus the lock starts out locked, and to wait for the
|
||||
// event you try to lock the lock. To signal the event,
|
||||
// you unlock the lock.
|
||||
|
||||
void
|
||||
runtime·noteclear(Note *n)
|
||||
{
|
||||
n->lock.key = 0; // memset(n, 0, sizeof *n)
|
||||
futexlock(&n->lock);
|
||||
n->state = 0;
|
||||
}
|
||||
|
||||
void
|
||||
runtime·notewakeup(Note *n)
|
||||
{
|
||||
futexunlock(&n->lock);
|
||||
runtime·xchg(&n->state, 1);
|
||||
futexwakeup(&n->state, 1<<30);
|
||||
}
|
||||
|
||||
void
|
||||
runtime·notesleep(Note *n)
|
||||
{
|
||||
futexlock(&n->lock);
|
||||
futexunlock(&n->lock); // Let other sleepers find out too.
|
||||
while(runtime·atomicload(&n->state) == 0)
|
||||
futexsleep(&n->state, 0);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue