mirror of
https://github.com/golang/go.git
synced 2025-12-08 06:10:04 +00:00
runtime: aggregate defer allocations
benchmark old ns/op new ns/op delta BenchmarkDefer 165 113 -31.52% BenchmarkDefer10 155 103 -33.55% BenchmarkDeferMany 216 158 -26.85% benchmark old allocs new allocs delta BenchmarkDefer 1 0 -100.00% BenchmarkDefer10 1 0 -100.00% BenchmarkDeferMany 1 0 -100.00% benchmark old bytes new bytes delta BenchmarkDefer 64 0 -100.00% BenchmarkDefer10 64 0 -100.00% BenchmarkDeferMany 64 66 3.12% Fixes #2364. R=ken2 CC=golang-dev https://golang.org/cl/7001051
This commit is contained in:
Russ Cox
parent
e09f1e7a46
commit
0de71619ce
4 changed files with 180 additions and 39 deletions
|
|
@ -121,7 +121,7 @@ runtime·cgocall(void (*fn)(void*), void *arg)
|
|||
* Lock g to m to ensure we stay on the same stack if we do a
|
||||
* cgo callback.
|
||||
*/
|
||||
d.nofree = false;
|
||||
d.special = false;
|
||||
if(m->lockedg == nil) {
|
||||
m->lockedg = g;
|
||||
g->lockedm = m;
|
||||
|
|
@ -131,7 +131,7 @@ runtime·cgocall(void (*fn)(void*), void *arg)
|
|||
d.siz = 0;
|
||||
d.link = g->defer;
|
||||
d.argp = (void*)-1; // unused because unlockm never recovers
|
||||
d.nofree = true;
|
||||
d.special = true;
|
||||
g->defer = &d;
|
||||
}
|
||||
|
||||
|
|
@ -160,7 +160,7 @@ runtime·cgocall(void (*fn)(void*), void *arg)
|
|||
m->cgomal = nil;
|
||||
}
|
||||
|
||||
if(d.nofree) {
|
||||
if(d.special) {
|
||||
if(g->defer != &d || d.fn != (byte*)unlockm)
|
||||
runtime·throw("runtime: bad defer entry in cgocallback");
|
||||
g->defer = d.link;
|
||||
|
|
@ -236,7 +236,7 @@ runtime·cgocallbackg(void (*fn)(void), void *arg, uintptr argsize)
|
|||
d.siz = 0;
|
||||
d.link = g->defer;
|
||||
d.argp = (void*)-1; // unused because unwindm never recovers
|
||||
d.nofree = true;
|
||||
d.special = true;
|
||||
g->defer = &d;
|
||||
|
||||
if(raceenabled)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,106 @@
|
|||
uint32 runtime·panicking;
|
||||
static Lock paniclk;
|
||||
|
||||
enum
|
||||
{
|
||||
DeferChunkSize = 2048
|
||||
};
|
||||
|
||||
// Allocate a Defer, usually as part of the larger frame of deferred functions.
|
||||
// Each defer must be released with both popdefer and freedefer.
|
||||
static Defer*
|
||||
newdefer(int32 siz)
|
||||
{
|
||||
int32 total;
|
||||
DeferChunk *c;
|
||||
Defer *d;
|
||||
|
||||
c = g->dchunk;
|
||||
total = sizeof(*d) + ROUND(siz, sizeof(uintptr)) - sizeof(d->args);
|
||||
if(c == nil || total > DeferChunkSize - c->off) {
|
||||
if(total > DeferChunkSize / 2) {
|
||||
// Not worth putting in any chunk.
|
||||
// Allocate a separate block.
|
||||
d = runtime·malloc(total);
|
||||
d->siz = siz;
|
||||
d->special = 1;
|
||||
d->free = 1;
|
||||
d->link = g->defer;
|
||||
g->defer = d;
|
||||
return d;
|
||||
}
|
||||
|
||||
// Cannot fit in current chunk.
|
||||
// Switch to next chunk, allocating if necessary.
|
||||
c = g->dchunknext;
|
||||
if(c == nil)
|
||||
c = runtime·malloc(DeferChunkSize);
|
||||
c->prev = g->dchunk;
|
||||
c->off = sizeof(*c);
|
||||
g->dchunk = c;
|
||||
g->dchunknext = nil;
|
||||
}
|
||||
|
||||
d = (Defer*)((byte*)c + c->off);
|
||||
c->off += total;
|
||||
d->siz = siz;
|
||||
d->special = 0;
|
||||
d->free = 0;
|
||||
d->link = g->defer;
|
||||
g->defer = d;
|
||||
return d;
|
||||
}
|
||||
|
||||
// Pop the current defer from the defer stack.
|
||||
// Its contents are still valid until the goroutine begins executing again.
|
||||
// In particular it is safe to call reflect.call(d->fn, d->argp, d->siz) after
|
||||
// popdefer returns.
|
||||
static void
|
||||
popdefer(void)
|
||||
{
|
||||
Defer *d;
|
||||
DeferChunk *c;
|
||||
int32 total;
|
||||
|
||||
d = g->defer;
|
||||
if(d == nil)
|
||||
runtime·throw("runtime: popdefer nil");
|
||||
g->defer = d->link;
|
||||
if(d->special) {
|
||||
// Nothing else to do.
|
||||
return;
|
||||
}
|
||||
total = sizeof(*d) + ROUND(d->siz, sizeof(uintptr)) - sizeof(d->args);
|
||||
c = g->dchunk;
|
||||
if(c == nil || (byte*)d+total != (byte*)c+c->off)
|
||||
runtime·throw("runtime: popdefer phase error");
|
||||
c->off -= total;
|
||||
if(c->off == sizeof(*c)) {
|
||||
// Chunk now empty, so pop from stack.
|
||||
// Save in dchunknext both to help with pingponging between frames
|
||||
// and to make sure d is still valid on return.
|
||||
if(g->dchunknext != nil)
|
||||
runtime·free(g->dchunknext);
|
||||
g->dchunknext = c;
|
||||
g->dchunk = c->prev;
|
||||
}
|
||||
}
|
||||
|
||||
// Free the given defer.
|
||||
// For defers in the per-goroutine chunk this just clears the saved arguments.
|
||||
// For large defers allocated on the heap, this frees them.
|
||||
// The defer cannot be used after this call.
|
||||
static void
|
||||
freedefer(Defer *d)
|
||||
{
|
||||
if(d->special) {
|
||||
if(d->free)
|
||||
runtime·free(d);
|
||||
} else {
|
||||
runtime·memclr((byte*)d->args, d->siz);
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new deferred function fn with siz bytes of arguments.
|
||||
// The compiler turns a defer statement into a call to this.
|
||||
// Cannot split the stack because it assumes that the arguments
|
||||
|
|
@ -22,14 +122,9 @@ uintptr
|
|||
runtime·deferproc(int32 siz, byte* fn, ...)
|
||||
{
|
||||
Defer *d;
|
||||
int32 mallocsiz;
|
||||
|
||||
mallocsiz = sizeof(*d);
|
||||
if(siz > sizeof(d->args))
|
||||
mallocsiz += siz - sizeof(d->args);
|
||||
d = runtime·malloc(mallocsiz);
|
||||
d = newdefer(siz);
|
||||
d->fn = fn;
|
||||
d->siz = siz;
|
||||
d->pc = runtime·getcallerpc(&siz);
|
||||
if(thechar == '5')
|
||||
d->argp = (byte*)(&fn+2); // skip caller's saved link register
|
||||
|
|
@ -37,9 +132,6 @@ runtime·deferproc(int32 siz, byte* fn, ...)
|
|||
d->argp = (byte*)(&fn+1);
|
||||
runtime·memmove(d->args, d->argp, d->siz);
|
||||
|
||||
d->link = g->defer;
|
||||
g->defer = d;
|
||||
|
||||
// deferproc returns 0 normally.
|
||||
// a deferred func that stops a panic
|
||||
// makes the deferproc return 1.
|
||||
|
|
@ -73,10 +165,9 @@ runtime·deferreturn(uintptr arg0)
|
|||
if(d->argp != argp)
|
||||
return;
|
||||
runtime·memmove(argp, d->args, d->siz);
|
||||
g->defer = d->link;
|
||||
fn = d->fn;
|
||||
if(!d->nofree)
|
||||
runtime·free(d);
|
||||
popdefer();
|
||||
freedefer(d);
|
||||
runtime·jmpdefer(fn, argp);
|
||||
}
|
||||
|
||||
|
|
@ -87,10 +178,9 @@ rundefer(void)
|
|||
Defer *d;
|
||||
|
||||
while((d = g->defer) != nil) {
|
||||
g->defer = d->link;
|
||||
popdefer();
|
||||
reflect·call(d->fn, (byte*)d->args, d->siz);
|
||||
if(!d->nofree)
|
||||
runtime·free(d);
|
||||
freedefer(d);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -117,6 +207,7 @@ runtime·panic(Eface e)
|
|||
{
|
||||
Defer *d;
|
||||
Panic *p;
|
||||
void *pc, *argp;
|
||||
|
||||
p = runtime·mal(sizeof *p);
|
||||
p->arg = e;
|
||||
|
|
@ -129,23 +220,23 @@ runtime·panic(Eface e)
|
|||
if(d == nil)
|
||||
break;
|
||||
// take defer off list in case of recursive panic
|
||||
g->defer = d->link;
|
||||
popdefer();
|
||||
g->ispanic = true; // rock for newstack, where reflect.call ends up
|
||||
argp = d->argp;
|
||||
pc = d->pc;
|
||||
reflect·call(d->fn, (byte*)d->args, d->siz);
|
||||
freedefer(d);
|
||||
if(p->recovered) {
|
||||
g->panic = p->link;
|
||||
if(g->panic == nil) // must be done with signal
|
||||
g->sig = 0;
|
||||
runtime·free(p);
|
||||
// put recovering defer back on list
|
||||
// for scheduler to find.
|
||||
d->link = g->defer;
|
||||
g->defer = d;
|
||||
// Pass information about recovering frame to recovery.
|
||||
g->sigcode0 = (uintptr)argp;
|
||||
g->sigcode1 = (uintptr)pc;
|
||||
runtime·mcall(recovery);
|
||||
runtime·throw("recovery failed"); // mcall should not return
|
||||
}
|
||||
if(!d->nofree)
|
||||
runtime·free(d);
|
||||
}
|
||||
|
||||
// ran out of deferred calls - old-school panic now
|
||||
|
|
@ -160,14 +251,15 @@ runtime·panic(Eface e)
|
|||
static void
|
||||
recovery(G *gp)
|
||||
{
|
||||
Defer *d;
|
||||
void *argp;
|
||||
void *pc;
|
||||
|
||||
// Rewind gp's stack; we're running on m->g0's stack.
|
||||
d = gp->defer;
|
||||
gp->defer = d->link;
|
||||
// Info about defer passed in G struct.
|
||||
argp = (void*)gp->sigcode0;
|
||||
pc = (void*)gp->sigcode1;
|
||||
|
||||
// Unwind to the stack frame with d's arguments in it.
|
||||
runtime·unwindstack(gp, d->argp);
|
||||
runtime·unwindstack(gp, argp);
|
||||
|
||||
// Make the deferproc for this d return again,
|
||||
// this time returning 1. The calling function will
|
||||
|
|
@ -179,12 +271,10 @@ recovery(G *gp)
|
|||
// before it tests the return value.)
|
||||
// On the arm there are 2 saved LRs mixed in too.
|
||||
if(thechar == '5')
|
||||
gp->sched.sp = (uintptr)d->argp - 4*sizeof(uintptr);
|
||||
gp->sched.sp = (uintptr)argp - 4*sizeof(uintptr);
|
||||
else
|
||||
gp->sched.sp = (uintptr)d->argp - 2*sizeof(uintptr);
|
||||
gp->sched.pc = d->pc;
|
||||
if(!d->nofree)
|
||||
runtime·free(d);
|
||||
gp->sched.sp = (uintptr)argp - 2*sizeof(uintptr);
|
||||
gp->sched.pc = pc;
|
||||
runtime·gogo(&gp->sched, 1);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ typedef struct Type Type;
|
|||
typedef struct ChanType ChanType;
|
||||
typedef struct MapType MapType;
|
||||
typedef struct Defer Defer;
|
||||
typedef struct DeferChunk DeferChunk;
|
||||
typedef struct Panic Panic;
|
||||
typedef struct Hmap Hmap;
|
||||
typedef struct Hchan Hchan;
|
||||
|
|
@ -218,6 +219,8 @@ struct G
|
|||
int32 sig;
|
||||
int32 writenbuf;
|
||||
byte* writebuf;
|
||||
DeferChunk *dchunk;
|
||||
DeferChunk *dchunknext;
|
||||
uintptr sigcode0;
|
||||
uintptr sigcode1;
|
||||
uintptr sigpc;
|
||||
|
|
@ -518,7 +521,8 @@ void runtime·nilintercopy(uintptr, void*, void*);
|
|||
struct Defer
|
||||
{
|
||||
int32 siz;
|
||||
bool nofree;
|
||||
bool special; // not part of defer frame
|
||||
bool free; // if special, free when done
|
||||
byte* argp; // where args were copied from
|
||||
byte* pc;
|
||||
byte* fn;
|
||||
|
|
@ -526,6 +530,12 @@ struct Defer
|
|||
void* args[1]; // padded to actual size
|
||||
};
|
||||
|
||||
struct DeferChunk
|
||||
{
|
||||
DeferChunk *prev;
|
||||
uintptr off;
|
||||
};
|
||||
|
||||
/*
|
||||
* panics
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -38,3 +38,44 @@ func BenchmarkIfaceCmpNil100(b *testing.B) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDefer(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
defer1()
|
||||
}
|
||||
}
|
||||
|
||||
func defer1() {
|
||||
defer func(x, y, z int) {
|
||||
if recover() != nil || x != 1 || y != 2 || z != 3 {
|
||||
panic("bad recover")
|
||||
}
|
||||
}(1, 2, 3)
|
||||
return
|
||||
}
|
||||
|
||||
func BenchmarkDefer10(b *testing.B) {
|
||||
for i := 0; i < b.N/10; i++ {
|
||||
defer2()
|
||||
}
|
||||
}
|
||||
|
||||
func defer2() {
|
||||
for i := 0; i < 10; i++ {
|
||||
defer func(x, y, z int) {
|
||||
if recover() != nil || x != 1 || y != 2 || z != 3 {
|
||||
panic("bad recover")
|
||||
}
|
||||
}(1, 2, 3)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDeferMany(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
defer func(x, y, z int) {
|
||||
if recover() != nil || x != 1 || y != 2 || z != 3 {
|
||||
panic("bad recover")
|
||||
}
|
||||
}(1, 2, 3)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue