mirror of
https://github.com/golang/go.git
synced 2025-12-08 06:10:04 +00:00
runtime: assume precisestack, copystack, StackCopyAlways, ScanStackByFrames
Commit to stack copying for stack growth. We're carrying around a surprising amount of cruft from older schemes. I am confident that precise stack scans and stack copying are here to stay. Delete fallback code for when precise stack info is disabled. Delete fallback code for when copying stacks is disabled. Delete fallback code for when StackCopyAlways is disabled. Delete Stktop chain - there is only one stack segment now. Delete M.moreargp, M.moreargsize, M.moreframesize, M.cret. Delete G.writenbuf (unrelated, just dead). Delete runtime.lessstack, runtime.oldstack. Delete many amd64 morestack variants. Delete initialization of morestack frame/arg sizes (shortens split prologue!). Replace G's stackguard/stackbase/stack0/stacksize/ syscallstack/syscallguard/forkstackguard with simple stack bounds (lo, hi). Update liblink, runtime/cgo for adjustments to G. LGTM=khr R=khr, bradfitz CC=golang-codereviews, iant, r https://golang.org/cl/137410043
This commit is contained in:
Russ Cox
parent
d72029e3a3
commit
15b76ad94b
50 changed files with 359 additions and 2894 deletions
|
|
@ -25,8 +25,6 @@ enum
|
|||
StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free
|
||||
|
||||
StackCache = 1,
|
||||
|
||||
StackCopyAlways = 1, // expect to be able to copy stacks 100% of the time
|
||||
};
|
||||
|
||||
// Global pool of spans that have free stacks.
|
||||
|
|
@ -185,13 +183,12 @@ runtime·stackcache_clear(MCache *c)
|
|||
runtime·unlock(&stackpoolmu);
|
||||
}
|
||||
|
||||
void*
|
||||
runtime·stackalloc(G *gp, uint32 n)
|
||||
Stack
|
||||
runtime·stackalloc(uint32 n)
|
||||
{
|
||||
uint8 order;
|
||||
uint32 n2;
|
||||
void *v;
|
||||
Stktop *top;
|
||||
MLink *x;
|
||||
MSpan *s;
|
||||
MCache *c;
|
||||
|
|
@ -206,12 +203,11 @@ runtime·stackalloc(G *gp, uint32 n)
|
|||
if(StackDebug >= 1)
|
||||
runtime·printf("stackalloc %d\n", n);
|
||||
|
||||
gp->stacksize += n;
|
||||
if(runtime·debug.efence || StackFromSystem) {
|
||||
v = runtime·sysAlloc(ROUND(n, PageSize), &mstats.stacks_sys);
|
||||
if(v == nil)
|
||||
runtime·throw("out of memory (stackalloc)");
|
||||
return v;
|
||||
return (Stack){(uintptr)v, (uintptr)v+n};
|
||||
}
|
||||
|
||||
// Small stacks are allocated with a fixed-size free-list allocator.
|
||||
|
|
@ -249,32 +245,32 @@ runtime·stackalloc(G *gp, uint32 n)
|
|||
runtime·throw("out of memory");
|
||||
v = (byte*)(s->start<<PageShift);
|
||||
}
|
||||
top = (Stktop*)((byte*)v+n-sizeof(Stktop));
|
||||
runtime·memclr((byte*)top, sizeof(*top));
|
||||
|
||||
if(raceenabled)
|
||||
runtime·racemalloc(v, n);
|
||||
if(StackDebug >= 1)
|
||||
runtime·printf(" allocated %p\n", v);
|
||||
return v;
|
||||
return (Stack){(uintptr)v, (uintptr)v+n};
|
||||
}
|
||||
|
||||
void
|
||||
runtime·stackfree(G *gp, void *v, Stktop *top)
|
||||
runtime·stackfree(Stack stk)
|
||||
{
|
||||
uint8 order;
|
||||
uintptr n, n2;
|
||||
MSpan *s;
|
||||
MLink *x;
|
||||
MCache *c;
|
||||
void *v;
|
||||
|
||||
n = (uintptr)(top+1) - (uintptr)v;
|
||||
n = stk.hi - stk.lo;
|
||||
v = (void*)stk.lo;
|
||||
if(n & (n-1))
|
||||
runtime·throw("stack not a power of 2");
|
||||
if(StackDebug >= 1) {
|
||||
runtime·printf("stackfree %p %d\n", v, (int32)n);
|
||||
runtime·memclr(v, n); // for testing, clobber stack data
|
||||
}
|
||||
gp->stacksize -= n;
|
||||
if(runtime·debug.efence || StackFromSystem) {
|
||||
if(runtime·debug.efence || StackFaultOnFree)
|
||||
runtime·SysFault(v, n);
|
||||
|
|
@ -312,70 +308,6 @@ runtime·stackfree(G *gp, void *v, Stktop *top)
|
|||
}
|
||||
}
|
||||
|
||||
// Called from runtime·lessstack when returning from a function which
|
||||
// allocated a new stack segment. The function's return value is in
|
||||
// m->cret.
|
||||
void
|
||||
runtime·oldstack(void)
|
||||
{
|
||||
Stktop *top;
|
||||
uint32 argsize;
|
||||
byte *sp, *old;
|
||||
uintptr *src, *dst, *dstend;
|
||||
G *gp;
|
||||
int64 goid;
|
||||
int32 oldstatus;
|
||||
|
||||
if(StackCopyAlways)
|
||||
runtime·throw("unexpected call to oldstack");
|
||||
|
||||
gp = g->m->curg;
|
||||
top = (Stktop*)gp->stackbase;
|
||||
if(top == nil)
|
||||
runtime·throw("nil stackbase");
|
||||
old = (byte*)gp->stackguard - StackGuard;
|
||||
sp = (byte*)top;
|
||||
argsize = top->argsize;
|
||||
|
||||
if(StackDebug >= 1) {
|
||||
runtime·printf("runtime: oldstack gobuf={pc:%p sp:%p lr:%p} cret=%p argsize=%p\n",
|
||||
top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, (uintptr)g->m->cret, (uintptr)argsize);
|
||||
}
|
||||
|
||||
gp->sched = top->gobuf;
|
||||
gp->sched.ret = g->m->cret;
|
||||
g->m->cret = 0; // drop reference
|
||||
// gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
|
||||
// happens during a function call inside entersyscall.
|
||||
|
||||
oldstatus = runtime·readgstatus(gp);
|
||||
oldstatus &= ~Gscan;
|
||||
if(oldstatus != Grunning && oldstatus != Gsyscall) {
|
||||
runtime·printf("runtime: oldstack status=%d\n", oldstatus);
|
||||
runtime·throw("oldstack");
|
||||
}
|
||||
runtime·casgstatus(gp, oldstatus, Gcopystack);
|
||||
gp->waitreason = runtime·gostringnocopy((byte*)"stack unsplit");
|
||||
|
||||
if(argsize > 0) {
|
||||
sp -= argsize;
|
||||
dst = (uintptr*)top->argp;
|
||||
dstend = dst + argsize/sizeof(*dst);
|
||||
src = (uintptr*)sp;
|
||||
while(dst < dstend)
|
||||
*dst++ = *src++;
|
||||
}
|
||||
goid = top->gobuf.g->goid; // fault if g is bad, before gogo
|
||||
USED(goid);
|
||||
|
||||
gp->stackbase = top->stackbase;
|
||||
gp->stackguard = top->stackguard;
|
||||
gp->stackguard0 = gp->stackguard;
|
||||
runtime·stackfree(gp, old, top);
|
||||
runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Grunning or Gsyscall
|
||||
runtime·gogo(&gp->sched);
|
||||
}
|
||||
|
||||
uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for real
|
||||
|
||||
static uint8*
|
||||
|
|
@ -403,8 +335,7 @@ mapnames[] = {
|
|||
|
||||
typedef struct CopyableInfo CopyableInfo;
|
||||
struct CopyableInfo {
|
||||
byte *stk; // bottom address of segment
|
||||
byte *base; // top address of segment (including Stktop)
|
||||
Stack stk;
|
||||
int32 frames; // count of copyable frames (-1 = not copyable)
|
||||
};
|
||||
|
||||
|
|
@ -421,9 +352,9 @@ checkframecopy(Stkframe *frame, void *arg)
|
|||
cinfo = arg;
|
||||
f = frame->fn;
|
||||
if(StackDebug >= 2)
|
||||
runtime·printf(" checking %s frame=[%p,%p] stk=[%p,%p]\n", runtime·funcname(f), frame->sp, frame->fp, cinfo->stk, cinfo->base);
|
||||
runtime·printf(" checking %s frame=[%p,%p] stk=[%p,%p]\n", runtime·funcname(f), frame->sp, frame->fp, cinfo->stk.lo, cinfo->stk.hi);
|
||||
// if we're not in the segment any more, return immediately.
|
||||
if((byte*)frame->varp < cinfo->stk || (byte*)frame->varp >= cinfo->base) {
|
||||
if(frame->varp < cinfo->stk.lo || frame->varp >= cinfo->stk.hi) {
|
||||
if(StackDebug >= 2)
|
||||
runtime·printf(" <next segment>\n");
|
||||
return false; // stop traceback
|
||||
|
|
@ -439,14 +370,12 @@ checkframecopy(Stkframe *frame, void *arg)
|
|||
stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
|
||||
if(stackmap == nil) {
|
||||
cinfo->frames = -1;
|
||||
if(StackDebug >= 1 || StackCopyAlways)
|
||||
runtime·printf("runtime: copystack: no locals info for %s\n", runtime·funcname(f));
|
||||
runtime·printf("runtime: copystack: no locals info for %s\n", runtime·funcname(f));
|
||||
return false;
|
||||
}
|
||||
if(stackmap->n <= 0) {
|
||||
cinfo->frames = -1;
|
||||
if(StackDebug >= 1 || StackCopyAlways)
|
||||
runtime·printf("runtime: copystack: locals size info only for %s\n", runtime·funcname(f));
|
||||
runtime·printf("runtime: copystack: locals size info only for %s\n", runtime·funcname(f));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -454,8 +383,7 @@ checkframecopy(Stkframe *frame, void *arg)
|
|||
stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
|
||||
if(stackmap == nil) {
|
||||
cinfo->frames = -1;
|
||||
if(StackDebug >= 1 || StackCopyAlways)
|
||||
runtime·printf("runtime: copystack: no arg info for %s\n", runtime·funcname(f));
|
||||
runtime·printf("runtime: copystack: no arg info for %s\n", runtime·funcname(f));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -476,10 +404,9 @@ copyabletopsegment(G *gp)
|
|||
StackMap *stackmap;
|
||||
bool (*cb)(Stkframe*, void*);
|
||||
|
||||
if(gp->stackbase == 0)
|
||||
runtime·throw("stackbase == 0");
|
||||
cinfo.stk = (byte*)gp->stackguard - StackGuard;
|
||||
cinfo.base = (byte*)gp->stackbase + sizeof(Stktop);
|
||||
if(gp->stack.lo == 0)
|
||||
runtime·throw("missing stack in copyabletopsegment");
|
||||
cinfo.stk = gp->stack;
|
||||
cinfo.frames = 0;
|
||||
|
||||
// Check that each frame is copyable. As a side effect,
|
||||
|
|
@ -494,21 +421,20 @@ copyabletopsegment(G *gp)
|
|||
|
||||
// Check to make sure all Defers are copyable
|
||||
for(d = gp->defer; d != nil; d = d->link) {
|
||||
if(cinfo.stk <= (byte*)d && (byte*)d < cinfo.base) {
|
||||
if(cinfo.stk.lo <= (uintptr)d && (uintptr)d < cinfo.stk.hi) {
|
||||
// Defer is on the stack. Its copyableness has
|
||||
// been established during stack walking.
|
||||
// For now, this only happens with the Defer in runtime.main.
|
||||
continue;
|
||||
}
|
||||
if((byte*)d->argp < cinfo.stk || cinfo.base <= (byte*)d->argp)
|
||||
if(d->argp < cinfo.stk.lo || cinfo.stk.hi <= d->argp)
|
||||
break; // a defer for the next segment
|
||||
fn = d->fn;
|
||||
if(fn == nil) // See issue 8047
|
||||
continue;
|
||||
f = runtime·findfunc((uintptr)fn->fn);
|
||||
if(f == nil) {
|
||||
if(StackDebug >= 1 || StackCopyAlways)
|
||||
runtime·printf("runtime: copystack: no func for deferred pc %p\n", fn->fn);
|
||||
runtime·printf("runtime: copystack: no func for deferred pc %p\n", fn->fn);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -519,18 +445,16 @@ copyabletopsegment(G *gp)
|
|||
// C (particularly, cgo) lies to us. See issue 7695.
|
||||
stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
|
||||
if(stackmap == nil || stackmap->n <= 0) {
|
||||
if(StackDebug >= 1 || StackCopyAlways)
|
||||
runtime·printf("runtime: copystack: no arg info for deferred %s\n", runtime·funcname(f));
|
||||
runtime·printf("runtime: copystack: no arg info for deferred %s\n", runtime·funcname(f));
|
||||
return -1;
|
||||
}
|
||||
stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
|
||||
if(stackmap == nil || stackmap->n <= 0) {
|
||||
if(StackDebug >= 1 || StackCopyAlways)
|
||||
runtime·printf("runtime: copystack: no local info for deferred %s\n", runtime·funcname(f));
|
||||
runtime·printf("runtime: copystack: no local info for deferred %s\n", runtime·funcname(f));
|
||||
return -1;
|
||||
}
|
||||
|
||||
if(cinfo.stk <= (byte*)fn && (byte*)fn < cinfo.base) {
|
||||
if(cinfo.stk.lo <= (uintptr)fn && (uintptr)fn < cinfo.stk.hi) {
|
||||
// FuncVal is on the stack. Again, its copyableness
|
||||
// was established during stack walking.
|
||||
continue;
|
||||
|
|
@ -548,8 +472,7 @@ copyabletopsegment(G *gp)
|
|||
|
||||
typedef struct AdjustInfo AdjustInfo;
|
||||
struct AdjustInfo {
|
||||
byte *oldstk; // bottom address of segment
|
||||
byte *oldbase; // top address of segment (after Stktop)
|
||||
Stack old;
|
||||
uintptr delta; // ptr distance from old to new stack (newbase - oldbase)
|
||||
};
|
||||
|
||||
|
|
@ -564,8 +487,8 @@ adjustpointers(byte **scanp, BitVector *bv, AdjustInfo *adjinfo, Func *f)
|
|||
Type *t;
|
||||
Itab *tab;
|
||||
|
||||
minp = adjinfo->oldstk;
|
||||
maxp = adjinfo->oldbase;
|
||||
minp = (byte*)adjinfo->old.lo;
|
||||
maxp = (byte*)adjinfo->old.hi;
|
||||
delta = adjinfo->delta;
|
||||
num = bv->n / BitsPerPointer;
|
||||
for(i = 0; i < num; i++) {
|
||||
|
|
@ -693,7 +616,7 @@ adjustframe(Stkframe *frame, void *arg)
|
|||
static void
|
||||
adjustctxt(G *gp, AdjustInfo *adjinfo)
|
||||
{
|
||||
if(adjinfo->oldstk <= (byte*)gp->sched.ctxt && (byte*)gp->sched.ctxt < adjinfo->oldbase)
|
||||
if(adjinfo->old.lo <= (uintptr)gp->sched.ctxt && (uintptr)gp->sched.ctxt < adjinfo->old.hi)
|
||||
gp->sched.ctxt = (byte*)gp->sched.ctxt + adjinfo->delta;
|
||||
}
|
||||
|
||||
|
|
@ -707,7 +630,7 @@ adjustdefers(G *gp, AdjustInfo *adjinfo)
|
|||
BitVector bv;
|
||||
|
||||
for(dp = &gp->defer, d = *dp; d != nil; dp = &d->link, d = *dp) {
|
||||
if(adjinfo->oldstk <= (byte*)d && (byte*)d < adjinfo->oldbase) {
|
||||
if(adjinfo->old.lo <= (uintptr)d && (uintptr)d < adjinfo->old.hi) {
|
||||
// The Defer record is on the stack. Its fields will
|
||||
// get adjusted appropriately.
|
||||
// This only happens for runtime.main and runtime.gopanic now,
|
||||
|
|
@ -719,8 +642,12 @@ adjustdefers(G *gp, AdjustInfo *adjinfo)
|
|||
*dp = (Defer*)((byte*)d + adjinfo->delta);
|
||||
continue;
|
||||
}
|
||||
if((byte*)d->argp < adjinfo->oldstk || adjinfo->oldbase <= (byte*)d->argp)
|
||||
break; // a defer for the next segment
|
||||
if(d->argp == NoArgs)
|
||||
continue;
|
||||
if(d->argp < adjinfo->old.lo || adjinfo->old.hi <= d->argp) {
|
||||
runtime·printf("runtime: adjustdefers argp=%p stk=%p %p\n", d->argp, adjinfo->old.lo, adjinfo->old.hi);
|
||||
runtime·throw("adjustdefers: unexpected argp");
|
||||
}
|
||||
fn = d->fn;
|
||||
if(fn == nil) {
|
||||
// Defer of nil function. It will panic when run, and there
|
||||
|
|
@ -734,7 +661,7 @@ adjustdefers(G *gp, AdjustInfo *adjinfo)
|
|||
if(StackDebug >= 4)
|
||||
runtime·printf(" checking defer %s\n", runtime·funcname(f));
|
||||
// Defer's FuncVal might be on the stack
|
||||
if(adjinfo->oldstk <= (byte*)fn && (byte*)fn < adjinfo->oldbase) {
|
||||
if(adjinfo->old.lo <= (uintptr)fn && (uintptr)fn < adjinfo->old.hi) {
|
||||
if(StackDebug >= 3)
|
||||
runtime·printf(" adjust defer fn %s\n", runtime·funcname(f));
|
||||
d->fn = (FuncVal*)((byte*)fn + adjinfo->delta);
|
||||
|
|
@ -758,7 +685,7 @@ adjustpanics(G *gp, AdjustInfo *adjinfo)
|
|||
// Panic structs are all on the stack
|
||||
// and are adjusted by stack copying.
|
||||
// The only pointer we need to update is gp->panic, the head of the list.
|
||||
if(adjinfo->oldstk <= (byte*)gp->panic && (byte*)gp->panic < adjinfo->oldbase)
|
||||
if(adjinfo->old.lo <= (uintptr)gp->panic && (uintptr)gp->panic < adjinfo->old.hi)
|
||||
gp->panic = (Panic*)((byte*)gp->panic + adjinfo->delta);
|
||||
}
|
||||
|
||||
|
|
@ -772,10 +699,10 @@ adjustsudogs(G *gp, AdjustInfo *adjinfo)
|
|||
// might be in the stack.
|
||||
for(s = gp->waiting; s != nil; s = s->waitlink) {
|
||||
e = s->elem;
|
||||
if(adjinfo->oldstk <= e && e < adjinfo->oldbase)
|
||||
if(adjinfo->old.lo <= (uintptr)e && (uintptr)e < adjinfo->old.hi)
|
||||
s->elem = e + adjinfo->delta;
|
||||
e = (byte*)s->selectdone;
|
||||
if(adjinfo->oldstk <= e && e < adjinfo->oldbase)
|
||||
if(adjinfo->old.lo <= (uintptr)e && (uintptr)e < adjinfo->old.hi)
|
||||
s->selectdone = (uint32*)(e + adjinfo->delta);
|
||||
}
|
||||
}
|
||||
|
|
@ -785,36 +712,28 @@ adjustsudogs(G *gp, AdjustInfo *adjinfo)
|
|||
static void
|
||||
copystack(G *gp, uintptr nframes, uintptr newsize)
|
||||
{
|
||||
byte *oldstk, *oldbase, *newstk, *newbase;
|
||||
uintptr oldsize, used;
|
||||
Stack old, new;
|
||||
uintptr used;
|
||||
AdjustInfo adjinfo;
|
||||
Stktop *oldtop, *newtop;
|
||||
uint32 oldstatus;
|
||||
bool (*cb)(Stkframe*, void*);
|
||||
|
||||
if(gp->syscallstack != 0)
|
||||
runtime·throw("can't handle stack copy in syscall yet");
|
||||
oldstk = (byte*)gp->stackguard - StackGuard;
|
||||
if(gp->stackbase == 0)
|
||||
if(gp->syscallsp != 0)
|
||||
runtime·throw("stack growth not allowed in system call");
|
||||
old = gp->stack;
|
||||
if(old.lo == 0)
|
||||
runtime·throw("nil stackbase");
|
||||
oldbase = (byte*)gp->stackbase + sizeof(Stktop);
|
||||
oldsize = oldbase - oldstk;
|
||||
used = oldbase - (byte*)gp->sched.sp;
|
||||
oldtop = (Stktop*)gp->stackbase;
|
||||
used = old.hi - gp->sched.sp;
|
||||
|
||||
// allocate new stack
|
||||
newstk = runtime·stackalloc(gp, newsize);
|
||||
newbase = newstk + newsize;
|
||||
newtop = (Stktop*)(newbase - sizeof(Stktop));
|
||||
new = runtime·stackalloc(newsize);
|
||||
|
||||
if(StackDebug >= 1)
|
||||
runtime·printf("copystack gp=%p [%p %p]/%d -> [%p %p]/%d\n", gp, oldstk, oldbase, (int32)oldsize, newstk, newbase, (int32)newsize);
|
||||
USED(oldsize);
|
||||
runtime·printf("copystack gp=%p [%p %p %p]/%d -> [%p %p %p]/%d\n", gp, old.lo, old.hi-used, old.hi, (int32)(old.hi-old.lo), new.lo, new.hi-used, new.hi, (int32)newsize);
|
||||
|
||||
// adjust pointers in the to-be-copied frames
|
||||
adjinfo.oldstk = oldstk;
|
||||
adjinfo.oldbase = oldbase;
|
||||
adjinfo.delta = newbase - oldbase;
|
||||
adjinfo.old = old;
|
||||
adjinfo.delta = new.hi - old.hi;
|
||||
cb = adjustframe;
|
||||
runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, nframes, &cb, &adjinfo, false);
|
||||
|
||||
|
|
@ -824,26 +743,25 @@ copystack(G *gp, uintptr nframes, uintptr newsize)
|
|||
adjustpanics(gp, &adjinfo);
|
||||
adjustsudogs(gp, &adjinfo);
|
||||
|
||||
// copy the stack (including Stktop) to the new location
|
||||
runtime·memmove(newbase - used, oldbase - used, used);
|
||||
// copy the stack to the new location
|
||||
runtime·memmove((byte*)new.hi - used, (byte*)old.hi - used, used);
|
||||
|
||||
oldstatus = runtime·readgstatus(gp);
|
||||
oldstatus &= ~Gscan;
|
||||
if (oldstatus == Gwaiting || oldstatus == Grunnable)
|
||||
if(oldstatus == Gwaiting || oldstatus == Grunnable)
|
||||
runtime·casgstatus(gp, oldstatus, Gcopystack); // oldstatus is Gwaiting or Grunnable
|
||||
else
|
||||
runtime·throw("copystack: bad status, not Gwaiting or Grunnable");
|
||||
|
||||
// Swap out old stack for new one
|
||||
gp->stackbase = (uintptr)newtop;
|
||||
gp->stackguard = (uintptr)newstk + StackGuard;
|
||||
gp->stackguard0 = (uintptr)newstk + StackGuard; // NOTE: might clobber a preempt request
|
||||
if(gp->stack0 == (uintptr)oldstk)
|
||||
gp->stack0 = (uintptr)newstk;
|
||||
gp->sched.sp = (uintptr)(newbase - used);
|
||||
gp->stack = new;
|
||||
gp->stackguard0 = new.lo + StackGuard; // NOTE: might clobber a preempt request
|
||||
gp->sched.sp = new.hi - used;
|
||||
|
||||
runtime·casgstatus(gp, Gcopystack, oldstatus); // oldstatus is Gwaiting or Grunnable
|
||||
|
||||
// free old stack
|
||||
runtime·stackfree(gp, oldstk, oldtop);
|
||||
runtime·stackfree(old);
|
||||
}
|
||||
|
||||
// round x up to a power of 2.
|
||||
|
|
@ -858,27 +776,22 @@ runtime·round2(int32 x)
|
|||
return 1 << s;
|
||||
}
|
||||
|
||||
// Called from runtime·morestack when a new stack segment is needed.
|
||||
// Allocate a new stack big enough for m->moreframesize bytes,
|
||||
// copy m->moreargsize bytes to the new frame,
|
||||
// and then act as though runtime·lessstack called the function at m->morepc.
|
||||
// Called from runtime·morestack when more stack is needed.
|
||||
// Allocate larger stack and relocate to new stack.
|
||||
// Stack growth is multiplicative, for constant amortized cost.
|
||||
//
|
||||
// g->atomicstatus will be Grunning, Gsyscall or Gscanrunning, Gscansyscall upon entry.
|
||||
// g->atomicstatus will be Grunning or Gscanrunning upon entry.
|
||||
// If the GC is trying to stop this g then it will set preemptscan to true.
|
||||
void
|
||||
runtime·newstack(void)
|
||||
{
|
||||
int32 framesize, argsize, oldstatus, oldsize, newsize, nframes;
|
||||
Stktop *top;
|
||||
byte *stk, *oldstk, *oldbase;
|
||||
int32 oldsize, newsize, nframes;
|
||||
uintptr sp;
|
||||
uintptr *src, *dst, *dstend;
|
||||
G *gp;
|
||||
Gobuf label, morebuf;
|
||||
void *moreargp;
|
||||
Gobuf morebuf;
|
||||
|
||||
if(g->m->forkstackguard)
|
||||
runtime·throw("split stack after fork");
|
||||
if(g->m->morebuf.g->stackguard0 == (uintptr)StackFork)
|
||||
runtime·throw("stack growth after fork");
|
||||
if(g->m->morebuf.g != g->m->curg) {
|
||||
runtime·printf("runtime: newstack called from g=%p\n"
|
||||
"\tm=%p m->curg=%p m->g0=%p m->gsignal=%p\n",
|
||||
|
|
@ -890,64 +803,50 @@ runtime·newstack(void)
|
|||
if(g->throwsplit)
|
||||
runtime·throw("runtime: stack split at bad time");
|
||||
|
||||
// The goroutine must be executing in order to call newstack, so the possible states are
|
||||
// Grunning and Gsyscall (and, due to GC, also Gscanrunning and Gscansyscall).
|
||||
// The goroutine must be executing in order to call newstack,
|
||||
// so it must be Grunning or Gscanrunning.
|
||||
|
||||
// gp->status is usually Grunning, but it could be Gsyscall if a stack overflow
|
||||
// happens during a function call inside entersyscall.
|
||||
gp = g->m->curg;
|
||||
oldstatus = runtime·readgstatus(gp) & ~Gscan;
|
||||
framesize = g->m->moreframesize;
|
||||
argsize = g->m->moreargsize;
|
||||
moreargp = g->m->moreargp;
|
||||
g->m->moreargp = nil;
|
||||
morebuf = g->m->morebuf;
|
||||
g->m->morebuf.pc = (uintptr)nil;
|
||||
g->m->morebuf.lr = (uintptr)nil;
|
||||
g->m->morebuf.sp = (uintptr)nil;
|
||||
|
||||
runtime·casgstatus(gp, oldstatus, Gwaiting); // oldstatus is not in a Gscan status
|
||||
runtime·casgstatus(gp, Grunning, Gwaiting);
|
||||
gp->waitreason = runtime·gostringnocopy((byte*)"stack growth");
|
||||
|
||||
runtime·rewindmorestack(&gp->sched);
|
||||
|
||||
if(gp->stackbase == 0)
|
||||
runtime·throw("nil stackbase");
|
||||
if(gp->stack.lo == 0)
|
||||
runtime·throw("missing stack in newstack");
|
||||
sp = gp->sched.sp;
|
||||
if(thechar == '6' || thechar == '8') {
|
||||
// The call to morestack cost a word.
|
||||
sp -= sizeof(uintreg);
|
||||
}
|
||||
if(StackDebug >= 1 || sp < gp->stackguard - StackGuard) {
|
||||
runtime·printf("runtime: newstack framesize=%p argsize=%p sp=%p stack=[%p, %p]\n"
|
||||
if(StackDebug >= 1 || sp < gp->stack.lo) {
|
||||
runtime·printf("runtime: newstack sp=%p stack=[%p, %p]\n"
|
||||
"\tmorebuf={pc:%p sp:%p lr:%p}\n"
|
||||
"\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n",
|
||||
(uintptr)framesize, (uintptr)argsize, sp, gp->stackguard - StackGuard, gp->stackbase,
|
||||
sp, gp->stack.lo, gp->stack.hi,
|
||||
g->m->morebuf.pc, g->m->morebuf.sp, g->m->morebuf.lr,
|
||||
gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt);
|
||||
}
|
||||
if(sp < gp->stackguard - StackGuard) {
|
||||
runtime·printf("runtime: gp=%p, gp->status=%d, oldstatus=%d\n ", (void*)gp, runtime·readgstatus(gp), oldstatus);
|
||||
runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stackguard - StackGuard);
|
||||
if(sp < gp->stack.lo) {
|
||||
runtime·printf("runtime: gp=%p, gp->status=%d\n ", (void*)gp, runtime·readgstatus(gp));
|
||||
runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stack.lo);
|
||||
runtime·throw("runtime: split stack overflow");
|
||||
}
|
||||
|
||||
if(argsize % sizeof(uintptr) != 0) {
|
||||
runtime·printf("runtime: stack growth with misaligned argsize %d\n", argsize);
|
||||
runtime·throw("runtime: stack growth argsize");
|
||||
}
|
||||
|
||||
if(gp->stackguard0 == (uintptr)StackPreempt) {
|
||||
if(gp == g->m->g0)
|
||||
runtime·throw("runtime: preempt g0");
|
||||
if(oldstatus == Grunning && g->m->p == nil && g->m->locks == 0)
|
||||
if(g->m->p == nil && g->m->locks == 0)
|
||||
runtime·throw("runtime: g is running but p is not");
|
||||
if(oldstatus == Gsyscall && g->m->locks == 0)
|
||||
runtime·throw("runtime: stack growth during syscall");
|
||||
if(oldstatus == Grunning && gp->preemptscan) {
|
||||
if(gp->preemptscan) {
|
||||
runtime·gcphasework(gp);
|
||||
runtime·casgstatus(gp, Gwaiting, Grunning);
|
||||
gp->stackguard0 = gp->stackguard;
|
||||
gp->stackguard0 = gp->stack.lo + StackGuard;
|
||||
gp->preempt = false;
|
||||
gp->preemptscan = false; // Tells the GC premption was successful.
|
||||
runtime·gogo(&gp->sched); // never return
|
||||
|
|
@ -955,105 +854,37 @@ runtime·newstack(void)
|
|||
|
||||
// Be conservative about where we preempt.
|
||||
// We are interested in preempting user Go code, not runtime code.
|
||||
if(oldstatus != Grunning || g->m->locks || g->m->mallocing || g->m->gcing || g->m->p->status != Prunning) {
|
||||
if(g->m->locks || g->m->mallocing || g->m->gcing || g->m->p->status != Prunning) {
|
||||
// Let the goroutine keep running for now.
|
||||
// gp->preempt is set, so it will be preempted next time.
|
||||
gp->stackguard0 = gp->stackguard;
|
||||
runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
|
||||
gp->stackguard0 = gp->stack.lo + StackGuard;
|
||||
runtime·casgstatus(gp, Gwaiting, Grunning);
|
||||
runtime·gogo(&gp->sched); // never return
|
||||
}
|
||||
// Act like goroutine called runtime.Gosched.
|
||||
runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
|
||||
runtime·casgstatus(gp, Gwaiting, Grunning);
|
||||
runtime·gosched_m(gp); // never return
|
||||
}
|
||||
|
||||
// If every frame on the top segment is copyable, allocate a bigger segment
|
||||
// and move the segment instead of allocating a new segment.
|
||||
if(runtime·copystack) {
|
||||
if(!runtime·precisestack)
|
||||
runtime·throw("can't copy stacks without precise stacks");
|
||||
nframes = copyabletopsegment(gp);
|
||||
if(nframes != -1) {
|
||||
oldstk = (byte*)gp->stackguard - StackGuard;
|
||||
oldbase = (byte*)gp->stackbase + sizeof(Stktop);
|
||||
oldsize = oldbase - oldstk;
|
||||
newsize = oldsize * 2;
|
||||
// Note that the concurrent GC might be scanning the stack as we try to replace it.
|
||||
// copystack takes care of the appropriate coordination with the stack scanner.
|
||||
copystack(gp, nframes, newsize);
|
||||
if(StackDebug >= 1)
|
||||
runtime·printf("stack grow done\n");
|
||||
if(gp->stacksize > runtime·maxstacksize) {
|
||||
runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize);
|
||||
runtime·throw("stack overflow");
|
||||
}
|
||||
runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Gsyscall or Grunning
|
||||
runtime·gogo(&gp->sched);
|
||||
}
|
||||
// TODO: if stack is uncopyable because we're in C code, patch return value at
|
||||
// end of C code to trigger a copy as soon as C code exits. That way, we'll
|
||||
// have stack available if we get this deep again.
|
||||
}
|
||||
// Allocate a bigger segment and move the stack.
|
||||
nframes = copyabletopsegment(gp);
|
||||
if(nframes == -1)
|
||||
runtime·throw("unable to grow stack");
|
||||
|
||||
if(StackCopyAlways)
|
||||
runtime·throw("split stack not allowed");
|
||||
|
||||
// allocate new segment.
|
||||
framesize += argsize;
|
||||
framesize += StackExtra; // room for more functions, Stktop.
|
||||
if(framesize < StackMin)
|
||||
framesize = StackMin;
|
||||
framesize += StackSystem;
|
||||
framesize = runtime·round2(framesize);
|
||||
stk = runtime·stackalloc(gp, framesize);
|
||||
if(gp->stacksize > runtime·maxstacksize) {
|
||||
oldsize = gp->stack.hi - gp->stack.lo;
|
||||
newsize = oldsize * 2;
|
||||
if(newsize > runtime·maxstacksize) {
|
||||
runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize);
|
||||
runtime·throw("stack overflow");
|
||||
}
|
||||
top = (Stktop*)(stk+framesize-sizeof(*top));
|
||||
|
||||
if(StackDebug >= 1) {
|
||||
runtime·printf("\t-> new stack gp=%p [%p, %p]\n", gp, stk, top);
|
||||
}
|
||||
|
||||
top->stackbase = gp->stackbase;
|
||||
top->stackguard = gp->stackguard;
|
||||
top->gobuf = morebuf;
|
||||
top->argp = moreargp;
|
||||
top->argsize = argsize;
|
||||
|
||||
gp->stackbase = (uintptr)top;
|
||||
gp->stackguard = (uintptr)stk + StackGuard;
|
||||
gp->stackguard0 = gp->stackguard;
|
||||
|
||||
sp = (uintptr)top;
|
||||
if(argsize > 0) {
|
||||
sp -= argsize;
|
||||
dst = (uintptr*)sp;
|
||||
dstend = dst + argsize/sizeof(*dst);
|
||||
src = (uintptr*)top->argp;
|
||||
while(dst < dstend)
|
||||
*dst++ = *src++;
|
||||
}
|
||||
|
||||
if(thechar == '5') {
|
||||
// caller would have saved its LR below args.
|
||||
sp -= sizeof(void*);
|
||||
*(void**)sp = nil;
|
||||
}
|
||||
|
||||
// Continue as if lessstack had just called m->morepc
|
||||
// (the PC that decided to grow the stack).
|
||||
runtime·memclr((byte*)&label, sizeof label);
|
||||
label.sp = sp;
|
||||
label.pc = (uintptr)runtime·lessstack;
|
||||
label.g = g->m->curg;
|
||||
runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt);
|
||||
gp->sched.ctxt = nil;
|
||||
runtime·casgstatus(gp, Gwaiting, oldstatus); // oldstatus is Grunning or Gsyscall
|
||||
runtime·gogo(&label);
|
||||
|
||||
*(int32*)345 = 123; // never return
|
||||
// Note that the concurrent GC might be scanning the stack as we try to replace it.
|
||||
// copystack takes care of the appropriate coordination with the stack scanner.
|
||||
copystack(gp, nframes, newsize);
|
||||
if(StackDebug >= 1)
|
||||
runtime·printf("stack grow done\n");
|
||||
runtime·casgstatus(gp, Gwaiting, Grunning);
|
||||
runtime·gogo(&gp->sched);
|
||||
}
|
||||
|
||||
#pragma textflag NOSPLIT
|
||||
|
|
@ -1083,28 +914,24 @@ void
|
|||
runtime·shrinkstack(G *gp)
|
||||
{
|
||||
int32 nframes;
|
||||
byte *oldstk, *oldbase;
|
||||
uintptr used, oldsize, newsize;
|
||||
|
||||
if(!runtime·copystack)
|
||||
return;
|
||||
if(runtime·readgstatus(gp) == Gdead)
|
||||
return;
|
||||
if(gp->stackbase == 0)
|
||||
runtime·throw("stackbase == 0");
|
||||
if(gp->stack.lo == 0)
|
||||
runtime·throw("missing stack in shrinkstack");
|
||||
//return; // TODO: why does this happen?
|
||||
oldstk = (byte*)gp->stackguard - StackGuard;
|
||||
oldbase = (byte*)gp->stackbase + sizeof(Stktop);
|
||||
oldsize = oldbase - oldstk;
|
||||
oldsize = gp->stack.hi - gp->stack.lo;
|
||||
newsize = oldsize / 2;
|
||||
if(newsize < FixedStack)
|
||||
return; // don't shrink below the minimum-sized stack
|
||||
used = oldbase - (byte*)gp->sched.sp;
|
||||
used = gp->stack.hi - gp->sched.sp;
|
||||
if(used >= oldsize / 4)
|
||||
return; // still using at least 1/4 of the segment.
|
||||
|
||||
if(gp->syscallstack != (uintptr)nil) // TODO: can we handle this case?
|
||||
if(gp->syscallsp != 0) // TODO: can we handle this case?
|
||||
return;
|
||||
|
||||
#ifdef GOOS_windows
|
||||
if(gp->m != nil && gp->m->libcallsp != 0)
|
||||
return;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue