diff options
Diffstat (limited to 'src/pkg/runtime/stack.c')
| -rw-r--r-- | src/pkg/runtime/stack.c | 711 |
1 files changed, 638 insertions, 73 deletions
diff --git a/src/pkg/runtime/stack.c b/src/pkg/runtime/stack.c index 634706051..1680f004e 100644 --- a/src/pkg/runtime/stack.c +++ b/src/pkg/runtime/stack.c @@ -6,10 +6,21 @@ #include "arch_GOARCH.h" #include "malloc.h" #include "stack.h" +#include "funcdata.h" +#include "typekind.h" +#include "type.h" +#include "../../cmd/ld/textflag.h" enum { + // StackDebug == 0: no logging + // == 1: logging of per-stack operations + // == 2: logging of per-frame operations + // == 3: logging of per-word updates + // == 4: logging of per-word reads StackDebug = 0, + StackFromSystem = 0, // allocate stacks from system memory instead of the heap + StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free }; typedef struct StackCacheNode StackCacheNode; @@ -74,22 +85,37 @@ stackcacherelease(void) } void* -runtime·stackalloc(uint32 n) +runtime·stackalloc(G *gp, uint32 n) { uint32 pos; void *v; + bool malloced; + Stktop *top; // Stackalloc must be called on scheduler stack, so that we // never try to grow the stack during the code that stackalloc runs. // Doing so would cause a deadlock (issue 1547). if(g != m->g0) runtime·throw("stackalloc not on scheduler stack"); + if((n & (n-1)) != 0) + runtime·throw("stack size not a power of 2"); + if(StackDebug >= 1) + runtime·printf("stackalloc %d\n", n); + + gp->stacksize += n; + if(runtime·debug.efence || StackFromSystem) { + v = runtime·SysAlloc(ROUND(n, PageSize), &mstats.stacks_sys); + if(v == nil) + runtime·throw("out of memory (stackalloc)"); + return v; + } - // Stacks are usually allocated with a fixed-size free-list allocator, - // but if we need a stack of non-standard size, we fall back on malloc - // (assuming that inside malloc and GC all the stack frames are small, + // Minimum-sized stacks are allocated with a fixed-size free-list allocator, + // but if we need a stack of a bigger size, we fall back on malloc + // (assuming that inside malloc all the stack frames are small, // so that we do not deadlock). - if(n == FixedStack || m->mallocing || m->gcing) { + malloced = true; + if(n == FixedStack || m->mallocing) { if(n != FixedStack) { runtime·printf("stackalloc: in malloc, size=%d want %d\n", FixedStack, n); runtime·throw("stackalloc"); @@ -102,27 +128,46 @@ runtime·stackalloc(uint32 n) m->stackcachepos = pos; m->stackcachecnt--; m->stackinuse++; - return v; - } - return runtime·mallocgc(n, 0, FlagNoProfiling|FlagNoGC|FlagNoZero|FlagNoInvokeGC); + malloced = false; + } else + v = runtime·mallocgc(n, 0, FlagNoProfiling|FlagNoGC|FlagNoZero|FlagNoInvokeGC); + + top = (Stktop*)((byte*)v+n-sizeof(Stktop)); + runtime·memclr((byte*)top, sizeof(*top)); + top->malloced = malloced; + return v; } void -runtime·stackfree(void *v, uintptr n) +runtime·stackfree(G *gp, void *v, Stktop *top) { uint32 pos; - - if(n == FixedStack || m->mallocing || m->gcing) { - if(m->stackcachecnt == StackCacheSize) - stackcacherelease(); - pos = m->stackcachepos; - m->stackcache[pos] = v; - m->stackcachepos = (pos + 1) % StackCacheSize; - m->stackcachecnt++; - m->stackinuse--; + uintptr n; + + n = (uintptr)(top+1) - (uintptr)v; + if(StackDebug >= 1) + runtime·printf("stackfree %p %d\n", v, (int32)n); + gp->stacksize -= n; + if(runtime·debug.efence || StackFromSystem) { + if(runtime·debug.efence || StackFaultOnFree) + runtime·SysFault(v, n); + else + runtime·SysFree(v, n, &mstats.stacks_sys); + return; + } + if(top->malloced) { + runtime·free(v); return; } - runtime·free(v); + if(n != FixedStack) + runtime·throw("stackfree: bad fixed size"); + if(m->stackcachecnt == StackCacheSize) + stackcacherelease(); + pos = m->stackcachepos; + m->stackcache[pos] = v; + m->stackcachepos = (pos + 1) % StackCacheSize; + m->stackcachecnt++; + m->stackinuse--; } // Called from runtime·lessstack when returning from a function which @@ -145,12 +190,12 @@ runtime·oldstack(void) sp = (byte*)top; argsize = top->argsize; - if(StackDebug) { + if(StackDebug >= 1) { runtime·printf("runtime: oldstack gobuf={pc:%p sp:%p lr:%p} cret=%p argsize=%p\n", - top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, m->cret, (uintptr)argsize); + top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, (uintptr)m->cret, (uintptr)argsize); } - // gp->status is usually Grunning, but it could be Gsyscall if a stack split + // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow // happens during a function call inside entersyscall. oldstatus = gp->status; @@ -175,11 +220,7 @@ runtime·oldstack(void) gp->stackguard = top->stackguard; gp->stackguard0 = gp->stackguard; gp->panicwrap = top->panicwrap; - - if(top->free != 0) { - gp->stacksize -= top->free; - runtime·stackfree(old, top->free); - } + runtime·stackfree(gp, old, top); gp->status = oldstatus; runtime·gogo(&gp->sched); @@ -187,6 +228,435 @@ runtime·oldstack(void) uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for real +static uint8* +mapnames[] = { + (uint8*)"---", + (uint8*)"scalar", + (uint8*)"ptr", + (uint8*)"multi", +}; + +// Stack frame layout +// +// (x86) +// +------------------+ +// | args from caller | +// +------------------+ <- frame->argp +// | return address | +// +------------------+ <- frame->varp +// | locals | +// +------------------+ +// | args to callee | +// +------------------+ <- frame->sp +// +// (arm: TODO) + +typedef struct CopyableInfo CopyableInfo; +struct CopyableInfo { + byte *stk; // bottom address of segment + byte *base; // top address of segment (including Stktop) + int32 frames; // count of copyable frames (-1 = not copyable) +}; + +void runtime·main(void); + +static bool +checkframecopy(Stkframe *frame, void *arg) +{ + CopyableInfo *cinfo; + Func *f; + StackMap *stackmap; + + cinfo = arg; + f = frame->fn; + if(StackDebug >= 2) + runtime·printf(" checking %s frame=[%p,%p] stk=[%p,%p]\n", runtime·funcname(f), frame->sp, frame->fp, cinfo->stk, cinfo->base); + // if we're not in the segment any more, return immediately. + if(frame->varp < cinfo->stk || frame->varp >= cinfo->base) { + if(StackDebug >= 2) + runtime·printf(" <next segment>\n"); + return false; // stop traceback + } + if(f->entry == (uintptr)runtime·main) { + // A special routine at the TOS of the main routine. + // We will allow it to be copied even though we don't + // have full GC info for it (because it is written in C). + cinfo->frames++; + return false; // stop traceback + } + if(frame->varp != (byte*)frame->sp) { // not in prologue (and has at least one local or outarg) + stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); + if(stackmap == nil) { + cinfo->frames = -1; + if(StackDebug >= 1) + runtime·printf("copystack: no locals info for %s\n", runtime·funcname(f)); + return false; + } + if(stackmap->n <= 0) { + cinfo->frames = -1; + if(StackDebug >= 1) + runtime·printf("copystack: locals size info only for %s\n", runtime·funcname(f)); + return false; + } + } + if(frame->arglen != 0) { + stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); + if(stackmap == nil) { + cinfo->frames = -1; + if(StackDebug >= 1) + runtime·printf("copystack: no arg info for %s\n", runtime·funcname(f)); + return false; + } + } + cinfo->frames++; + return true; // this frame is ok; keep going +} + +// If the top segment of the stack contains an uncopyable +// frame, return -1. Otherwise return the number of frames +// in the top segment, all of which are copyable. +static int32 +copyabletopsegment(G *gp) +{ + CopyableInfo cinfo; + Defer *d; + Func *f; + FuncVal *fn; + StackMap *stackmap; + + cinfo.stk = (byte*)gp->stackguard - StackGuard; + cinfo.base = (byte*)gp->stackbase + sizeof(Stktop); + cinfo.frames = 0; + + // Check that each frame is copyable. As a side effect, + // count the frames. + runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, checkframecopy, &cinfo, false); + if(StackDebug >= 1 && cinfo.frames != -1) + runtime·printf("copystack: %d copyable frames\n", cinfo.frames); + + // Check to make sure all Defers are copyable + for(d = gp->defer; d != nil; d = d->link) { + if(cinfo.stk <= (byte*)d && (byte*)d < cinfo.base) { + // Defer is on the stack. Its copyableness has + // been established during stack walking. + // For now, this only happens with the Defer in runtime.main. + continue; + } + if(d->argp < cinfo.stk || cinfo.base <= d->argp) + break; // a defer for the next segment + fn = d->fn; + if(fn == nil) // See issue 8047 + continue; + f = runtime·findfunc((uintptr)fn->fn); + if(f == nil) + return -1; + + // Check to make sure we have an args pointer map for the defer's args. + // We only need the args map, but we check + // for the locals map also, because when the locals map + // isn't provided it means the ptr map came from C and + // C (particularly, cgo) lies to us. See issue 7695. + stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); + if(stackmap == nil || stackmap->n <= 0) + return -1; + stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); + if(stackmap == nil || stackmap->n <= 0) + return -1; + + if(cinfo.stk <= (byte*)fn && (byte*)fn < cinfo.base) { + // FuncVal is on the stack. Again, its copyableness + // was established during stack walking. + continue; + } + // The FuncVal may have pointers in it, but fortunately for us + // the compiler won't put pointers into the stack in a + // heap-allocated FuncVal. + // One day if we do need to check this, we'll need maps of the + // pointerness of the closure args. The only place we have that map + // right now is in the gc program for the FuncVal. Ugh. + } + + return cinfo.frames; +} + +typedef struct AdjustInfo AdjustInfo; +struct AdjustInfo { + byte *oldstk; // bottom address of segment + byte *oldbase; // top address of segment (after Stktop) + uintptr delta; // ptr distance from old to new stack (newbase - oldbase) +}; + +// bv describes the memory starting at address scanp. +// Adjust any pointers contained therein. +static void +adjustpointers(byte **scanp, BitVector *bv, AdjustInfo *adjinfo, Func *f) +{ + uintptr delta; + int32 num, i; + byte *p, *minp, *maxp; + Type *t; + Itab *tab; + + minp = adjinfo->oldstk; + maxp = adjinfo->oldbase; + delta = adjinfo->delta; + num = bv->n / BitsPerPointer; + for(i = 0; i < num; i++) { + if(StackDebug >= 4) + runtime·printf(" %p:%s:%p\n", &scanp[i], mapnames[bv->data[i / (32 / BitsPerPointer)] >> (i * BitsPerPointer & 31) & 3], scanp[i]); + switch(bv->data[i / (32 / BitsPerPointer)] >> (i * BitsPerPointer & 31) & 3) { + case BitsDead: + if(runtime·debug.gcdead) + scanp[i] = (byte*)PoisonStack; + break; + case BitsScalar: + break; + case BitsPointer: + p = scanp[i]; + if(f != nil && (byte*)0 < p && (p < (byte*)PageSize || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) { + // Looks like a junk value in a pointer slot. + // Live analysis wrong? + m->traceback = 2; + runtime·printf("runtime: bad pointer in frame %s at %p: %p\n", runtime·funcname(f), &scanp[i], p); + runtime·throw("bad pointer!"); + } + if(minp <= p && p < maxp) { + if(StackDebug >= 3) + runtime·printf("adjust ptr %p %s\n", p, runtime·funcname(f)); + scanp[i] = p + delta; + } + break; + case BitsMultiWord: + switch(bv->data[(i+1) / (32 / BitsPerPointer)] >> ((i+1) * BitsPerPointer & 31) & 3) { + case BitsString: + // string referents are never on the stack, never need to be adjusted + i++; // skip len + break; + case BitsSlice: + p = scanp[i]; + if(minp <= p && p < maxp) { + if(StackDebug >= 3) + runtime·printf("adjust slice %p\n", p); + scanp[i] = p + delta; + } + i += 2; // skip len, cap + break; + case BitsEface: + t = (Type*)scanp[i]; + if(t != nil && (t->size > PtrSize || (t->kind & KindNoPointers) == 0)) { + p = scanp[i+1]; + if(minp <= p && p < maxp) { + if(StackDebug >= 3) + runtime·printf("adjust eface %p\n", p); + if(t->size > PtrSize) // currently we always allocate such objects on the heap + runtime·throw("large interface value found on stack"); + scanp[i+1] = p + delta; + } + } + i++; + break; + case BitsIface: + tab = (Itab*)scanp[i]; + if(tab != nil) { + t = tab->type; + //runtime·printf(" type=%p\n", t); + if(t->size > PtrSize || (t->kind & KindNoPointers) == 0) { + p = scanp[i+1]; + if(minp <= p && p < maxp) { + if(StackDebug >= 3) + runtime·printf("adjust iface %p\n", p); + if(t->size > PtrSize) // currently we always allocate such objects on the heap + runtime·throw("large interface value found on stack"); + scanp[i+1] = p + delta; + } + } + } + i++; + break; + } + break; + } + } +} + +// Note: the argument/return area is adjusted by the callee. +static bool +adjustframe(Stkframe *frame, void *arg) +{ + AdjustInfo *adjinfo; + Func *f; + StackMap *stackmap; + int32 pcdata; + BitVector bv; + uintptr targetpc; + + adjinfo = arg; + f = frame->fn; + if(StackDebug >= 2) + runtime·printf(" adjusting %s frame=[%p,%p] pc=%p continpc=%p\n", runtime·funcname(f), frame->sp, frame->fp, frame->pc, frame->continpc); + if(f->entry == (uintptr)runtime·main) + return true; + targetpc = frame->continpc; + if(targetpc == 0) { + // Frame is dead. + return true; + } + if(targetpc != f->entry) + targetpc--; + pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); + if(pcdata == -1) + pcdata = 0; // in prologue + + // adjust local pointers + if(frame->varp != (byte*)frame->sp) { + stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); + if(stackmap == nil) + runtime·throw("no locals info"); + if(stackmap->n <= 0) + runtime·throw("locals size info only"); + bv = runtime·stackmapdata(stackmap, pcdata); + if(StackDebug >= 3) + runtime·printf(" locals\n"); + adjustpointers((byte**)frame->varp - bv.n / BitsPerPointer, &bv, adjinfo, f); + } + // adjust inargs and outargs + if(frame->arglen != 0) { + stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); + if(stackmap == nil) + runtime·throw("no arg info"); + bv = runtime·stackmapdata(stackmap, pcdata); + if(StackDebug >= 3) + runtime·printf(" args\n"); + adjustpointers((byte**)frame->argp, &bv, adjinfo, nil); + } + return true; +} + +static void +adjustctxt(G *gp, AdjustInfo *adjinfo) +{ + if(adjinfo->oldstk <= (byte*)gp->sched.ctxt && (byte*)gp->sched.ctxt < adjinfo->oldbase) + gp->sched.ctxt = (byte*)gp->sched.ctxt + adjinfo->delta; +} + +static void +adjustdefers(G *gp, AdjustInfo *adjinfo) +{ + Defer *d, **dp; + Func *f; + FuncVal *fn; + StackMap *stackmap; + BitVector bv; + + for(dp = &gp->defer, d = *dp; d != nil; dp = &d->link, d = *dp) { + if(adjinfo->oldstk <= (byte*)d && (byte*)d < adjinfo->oldbase) { + // The Defer record is on the stack. Its fields will + // get adjusted appropriately. + // This only happens for runtime.main now, but a compiler + // optimization could do more of this. + *dp = (Defer*)((byte*)d + adjinfo->delta); + continue; + } + if(d->argp < adjinfo->oldstk || adjinfo->oldbase <= d->argp) + break; // a defer for the next segment + fn = d->fn; + if(fn == nil) { + // Defer of nil function. It will panic when run, and there + // aren't any args to adjust. See issue 8047. + d->argp += adjinfo->delta; + continue; + } + f = runtime·findfunc((uintptr)fn->fn); + if(f == nil) + runtime·throw("can't adjust unknown defer"); + if(StackDebug >= 4) + runtime·printf(" checking defer %s\n", runtime·funcname(f)); + // Defer's FuncVal might be on the stack + if(adjinfo->oldstk <= (byte*)fn && (byte*)fn < adjinfo->oldbase) { + if(StackDebug >= 3) + runtime·printf(" adjust defer fn %s\n", runtime·funcname(f)); + d->fn = (FuncVal*)((byte*)fn + adjinfo->delta); + } else { + // deferred function's args might point into the stack. + if(StackDebug >= 3) + runtime·printf(" adjust deferred args for %s\n", runtime·funcname(f)); + stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); + if(stackmap == nil) + runtime·throw("runtime: deferred function has no arg ptr map"); + bv = runtime·stackmapdata(stackmap, 0); + adjustpointers(d->args, &bv, adjinfo, f); + } + d->argp += adjinfo->delta; + } +} + +// Copies the top stack segment of gp to a new stack segment of a +// different size. The top segment must contain nframes frames. +static void +copystack(G *gp, uintptr nframes, uintptr newsize) +{ + byte *oldstk, *oldbase, *newstk, *newbase; + uintptr oldsize, used; + AdjustInfo adjinfo; + Stktop *oldtop, *newtop; + bool malloced; + + if(gp->syscallstack != 0) + runtime·throw("can't handle stack copy in syscall yet"); + oldstk = (byte*)gp->stackguard - StackGuard; + oldbase = (byte*)gp->stackbase + sizeof(Stktop); + oldsize = oldbase - oldstk; + used = oldbase - (byte*)gp->sched.sp; + oldtop = (Stktop*)gp->stackbase; + + // allocate new stack + newstk = runtime·stackalloc(gp, newsize); + newbase = newstk + newsize; + newtop = (Stktop*)(newbase - sizeof(Stktop)); + malloced = newtop->malloced; + + if(StackDebug >= 1) + runtime·printf("copystack [%p %p]/%d -> [%p %p]/%d\n", oldstk, oldbase, (int32)oldsize, newstk, newbase, (int32)newsize); + USED(oldsize); + + // adjust pointers in the to-be-copied frames + adjinfo.oldstk = oldstk; + adjinfo.oldbase = oldbase; + adjinfo.delta = newbase - oldbase; + runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, nframes, adjustframe, &adjinfo, false); + + // adjust other miscellaneous things that have pointers into stacks. + adjustctxt(gp, &adjinfo); + adjustdefers(gp, &adjinfo); + + // copy the stack (including Stktop) to the new location + runtime·memmove(newbase - used, oldbase - used, used); + newtop->malloced = malloced; + + // Swap out old stack for new one + gp->stackbase = (uintptr)newtop; + gp->stackguard = (uintptr)newstk + StackGuard; + gp->stackguard0 = (uintptr)newstk + StackGuard; // NOTE: might clobber a preempt request + if(gp->stack0 == (uintptr)oldstk) + gp->stack0 = (uintptr)newstk; + gp->sched.sp = (uintptr)(newbase - used); + + // free old stack + runtime·stackfree(gp, oldstk, oldtop); +} + +// round x up to a power of 2. +int32 +runtime·round2(int32 x) +{ + int32 s; + + s = 0; + while((1 << s) < x) + s++; + return 1 << s; +} + // Called from runtime·newstackcall or from runtime·morestack when a new // stack segment is needed. Allocate a new stack big enough for // m->moreframesize bytes, copy m->moreargsize bytes to the new frame, @@ -195,16 +665,18 @@ uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for void runtime·newstack(void) { - int32 framesize, argsize, oldstatus; + int32 framesize, argsize, oldstatus, oldsize, newsize, nframes; Stktop *top, *oldtop; - byte *stk; + byte *stk, *oldstk, *oldbase; uintptr sp; uintptr *src, *dst, *dstend; G *gp; - Gobuf label; + Gobuf label, morebuf; + void *moreargp; bool newstackcall; - uintptr free; + if(m->forkstackguard) + runtime·throw("split stack after fork"); if(m->morebuf.g != m->curg) { runtime·printf("runtime: newstack called from g=%p\n" "\tm=%p m->curg=%p m->g0=%p m->gsignal=%p\n", @@ -212,15 +684,21 @@ runtime·newstack(void) runtime·throw("runtime: wrong goroutine in newstack"); } - // gp->status is usually Grunning, but it could be Gsyscall if a stack split + // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow // happens during a function call inside entersyscall. gp = m->curg; oldstatus = gp->status; framesize = m->moreframesize; argsize = m->moreargsize; + moreargp = m->moreargp; + m->moreargp = nil; + morebuf = m->morebuf; + m->morebuf.pc = (uintptr)nil; + m->morebuf.lr = (uintptr)nil; + m->morebuf.sp = (uintptr)nil; gp->status = Gwaiting; - gp->waitreason = "stack split"; + gp->waitreason = "stack growth"; newstackcall = framesize==1; if(newstackcall) framesize = 0; @@ -234,7 +712,7 @@ runtime·newstack(void) // The call to morestack cost a word. sp -= sizeof(uintptr); } - if(StackDebug || sp < gp->stackguard - StackGuard) { + if(StackDebug >= 1 || sp < gp->stackguard - StackGuard) { runtime·printf("runtime: newstack framesize=%p argsize=%p sp=%p stack=[%p, %p]\n" "\tmorebuf={pc:%p sp:%p lr:%p}\n" "\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n", @@ -248,8 +726,8 @@ runtime·newstack(void) } if(argsize % sizeof(uintptr) != 0) { - runtime·printf("runtime: stack split with misaligned argsize %d\n", argsize); - runtime·throw("runtime: stack split argsize"); + runtime·printf("runtime: stack growth with misaligned argsize %d\n", argsize); + runtime·throw("runtime: stack growth argsize"); } if(gp->stackguard0 == (uintptr)StackPreempt) { @@ -258,7 +736,7 @@ runtime·newstack(void) if(oldstatus == Grunning && m->p == nil && m->locks == 0) runtime·throw("runtime: g is running but p is not"); if(oldstatus == Gsyscall && m->locks == 0) - runtime·throw("runtime: stack split during syscall"); + runtime·throw("runtime: stack growth during syscall"); // Be conservative about where we preempt. // We are interested in preempting user Go code, not runtime code. if(oldstatus != Grunning || m->locks || m->mallocing || m->gcing || m->p->status != Prunning) { @@ -273,46 +751,55 @@ runtime·newstack(void) runtime·gosched0(gp); // never return } - if(newstackcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) { - // special case: called from runtime.newstackcall (framesize==1) - // to call code with an arbitrary argument size, - // and we have enough space on the current stack. - // the new Stktop* is necessary to unwind, but - // we don't need to create a new segment. - top = (Stktop*)(m->morebuf.sp - sizeof(*top)); - stk = (byte*)gp->stackguard - StackGuard; - free = 0; - } else { - // allocate new segment. - framesize += argsize; - framesize += StackExtra; // room for more functions, Stktop. - if(framesize < StackMin) - framesize = StackMin; - framesize += StackSystem; - gp->stacksize += framesize; - if(gp->stacksize > runtime·maxstacksize) { - runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); - runtime·throw("stack overflow"); + // If every frame on the top segment is copyable, allocate a bigger segment + // and move the segment instead of allocating a new segment. + if(runtime·copystack) { + if(!runtime·precisestack) + runtime·throw("can't copy stacks without precise stacks"); + nframes = copyabletopsegment(gp); + if(nframes != -1) { + oldstk = (byte*)gp->stackguard - StackGuard; + oldbase = (byte*)gp->stackbase + sizeof(Stktop); + oldsize = oldbase - oldstk; + newsize = oldsize * 2; + copystack(gp, nframes, newsize); + if(StackDebug >= 1) + runtime·printf("stack grow done\n"); + if(gp->stacksize > runtime·maxstacksize) { + runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); + runtime·throw("stack overflow"); + } + gp->status = oldstatus; + runtime·gogo(&gp->sched); } - stk = runtime·stackalloc(framesize); - top = (Stktop*)(stk+framesize-sizeof(*top)); - free = framesize; + // TODO: if stack is uncopyable because we're in C code, patch return value at + // end of C code to trigger a copy as soon as C code exits. That way, we'll + // have stack available if we get this deep again. } - if(StackDebug) { + // allocate new segment. + framesize += argsize; + framesize += StackExtra; // room for more functions, Stktop. + if(framesize < StackMin) + framesize = StackMin; + framesize += StackSystem; + framesize = runtime·round2(framesize); + stk = runtime·stackalloc(gp, framesize); + if(gp->stacksize > runtime·maxstacksize) { + runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); + runtime·throw("stack overflow"); + } + top = (Stktop*)(stk+framesize-sizeof(*top)); + + if(StackDebug >= 1) { runtime·printf("\t-> new stack [%p, %p]\n", stk, top); } top->stackbase = gp->stackbase; top->stackguard = gp->stackguard; - top->gobuf = m->morebuf; - top->argp = m->moreargp; + top->gobuf = morebuf; + top->argp = moreargp; top->argsize = argsize; - top->free = free; - m->moreargp = nil; - m->morebuf.pc = (uintptr)nil; - m->morebuf.lr = (uintptr)nil; - m->morebuf.sp = (uintptr)nil; // copy flag from panic top->panic = gp->ispanic; @@ -365,18 +852,96 @@ runtime·newstack(void) *(int32*)345 = 123; // never return } +#pragma textflag NOSPLIT +void +runtime·nilfunc(void) +{ + *(byte*)0 = 0; +} + // adjust Gobuf as if it executed a call to fn // and then did an immediate gosave. void runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv) { - runtime·gostartcall(gobuf, fv->fn, fv); + void *fn; + + if(fv != nil) + fn = fv->fn; + else + fn = runtime·nilfunc; + runtime·gostartcall(gobuf, fn, fv); } +// Maybe shrink the stack being used by gp. +// Called at garbage collection time. void -runtime∕debug·setMaxStack(intgo in, intgo out) +runtime·shrinkstack(G *gp) { - out = runtime·maxstacksize; - runtime·maxstacksize = in; - FLUSH(&out); + int32 nframes; + byte *oldstk, *oldbase; + uintptr used, oldsize, newsize; + MSpan *span; + + if(!runtime·copystack) + return; + oldstk = (byte*)gp->stackguard - StackGuard; + oldbase = (byte*)gp->stackbase + sizeof(Stktop); + oldsize = oldbase - oldstk; + newsize = oldsize / 2; + if(newsize < FixedStack) + return; // don't shrink below the minimum-sized stack + used = oldbase - (byte*)gp->sched.sp; + if(used >= oldsize / 4) + return; // still using at least 1/4 of the segment. + + // To shrink to less than 1/2 a page, we need to copy. + if(newsize < PageSize/2) { + if(gp->syscallstack != (uintptr)nil) // TODO: can we handle this case? + return; +#ifdef GOOS_windows + if(gp->m != nil && gp->m->libcallsp != 0) + return; +#endif + nframes = copyabletopsegment(gp); + if(nframes == -1) + return; + copystack(gp, nframes, newsize); + return; + } + + // To shrink a stack of one page size or more, we can shrink it + // without copying. Just deallocate the lower half. + span = runtime·MHeap_LookupMaybe(&runtime·mheap, oldstk); + if(span == nil) + return; // stack allocated outside heap. Can't shrink it. Can happen if stack is allocated while inside malloc. TODO: shrink by copying? + if(span->elemsize != oldsize) + runtime·throw("span element size doesn't match stack size"); + if((uintptr)oldstk != span->start << PageShift) + runtime·throw("stack not at start of span"); + + if(StackDebug) + runtime·printf("shrinking stack in place %p %X->%X\n", oldstk, oldsize, newsize); + + // new stack guard for smaller stack + gp->stackguard = (uintptr)oldstk + newsize + StackGuard; + gp->stackguard0 = (uintptr)oldstk + newsize + StackGuard; + if(gp->stack0 == (uintptr)oldstk) + gp->stack0 = (uintptr)oldstk + newsize; + gp->stacksize -= oldsize - newsize; + + // Free bottom half of the stack. + if(runtime·debug.efence || StackFromSystem) { + if(runtime·debug.efence || StackFaultOnFree) + runtime·SysFault(oldstk, newsize); + else + runtime·SysFree(oldstk, newsize, &mstats.stacks_sys); + return; + } + // First, we trick malloc into thinking + // we allocated the stack as two separate half-size allocs. Then the + // free() call does the rest of the work for us. + runtime·MSpan_EnsureSwept(span); + runtime·MHeap_SplitSpan(&runtime·mheap, span); + runtime·free(oldstk); } |