// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "runtime.h" #include "arch_GOARCH.h" #include "malloc.h" #include "stack.h" #include "funcdata.h" #include "typekind.h" #include "type.h" #include "../../cmd/ld/textflag.h" enum { // StackDebug == 0: no logging // == 1: logging of per-stack operations // == 2: logging of per-frame operations // == 3: logging of per-word updates // == 4: logging of per-word reads StackDebug = 0, StackFromSystem = 0, // allocate stacks from system memory instead of the heap StackFaultOnFree = 0, // old stacks are mapped noaccess to detect use after free }; typedef struct StackCacheNode StackCacheNode; struct StackCacheNode { StackCacheNode *next; void* batch[StackCacheBatch-1]; }; static StackCacheNode *stackcache; static Lock stackcachemu; // stackcacherefill/stackcacherelease implement a global cache of stack segments. // The cache is required to prevent unlimited growth of per-thread caches. static void stackcacherefill(void) { StackCacheNode *n; int32 i, pos; runtime·lock(&stackcachemu); n = stackcache; if(n) stackcache = n->next; runtime·unlock(&stackcachemu); if(n == nil) { n = (StackCacheNode*)runtime·SysAlloc(FixedStack*StackCacheBatch, &mstats.stacks_sys); if(n == nil) runtime·throw("out of memory (stackcacherefill)"); for(i = 0; i < StackCacheBatch-1; i++) n->batch[i] = (byte*)n + (i+1)*FixedStack; } pos = m->stackcachepos; for(i = 0; i < StackCacheBatch-1; i++) { m->stackcache[pos] = n->batch[i]; pos = (pos + 1) % StackCacheSize; } m->stackcache[pos] = n; pos = (pos + 1) % StackCacheSize; m->stackcachepos = pos; m->stackcachecnt += StackCacheBatch; } static void stackcacherelease(void) { StackCacheNode *n; uint32 i, pos; pos = (m->stackcachepos - m->stackcachecnt) % StackCacheSize; n = (StackCacheNode*)m->stackcache[pos]; pos = (pos + 1) % StackCacheSize; for(i = 0; i < StackCacheBatch-1; i++) { n->batch[i] = m->stackcache[pos]; pos = (pos + 1) % StackCacheSize; } m->stackcachecnt -= StackCacheBatch; runtime·lock(&stackcachemu); n->next = stackcache; stackcache = n; runtime·unlock(&stackcachemu); } void* runtime·stackalloc(G *gp, uint32 n) { uint32 pos; void *v; bool malloced; Stktop *top; // Stackalloc must be called on scheduler stack, so that we // never try to grow the stack during the code that stackalloc runs. // Doing so would cause a deadlock (issue 1547). if(g != m->g0) runtime·throw("stackalloc not on scheduler stack"); if((n & (n-1)) != 0) runtime·throw("stack size not a power of 2"); if(StackDebug >= 1) runtime·printf("stackalloc %d\n", n); gp->stacksize += n; if(runtime·debug.efence || StackFromSystem) { v = runtime·SysAlloc(ROUND(n, PageSize), &mstats.stacks_sys); if(v == nil) runtime·throw("out of memory (stackalloc)"); return v; } // Minimum-sized stacks are allocated with a fixed-size free-list allocator, // but if we need a stack of a bigger size, we fall back on malloc // (assuming that inside malloc all the stack frames are small, // so that we do not deadlock). malloced = true; if(n == FixedStack || m->mallocing) { if(n != FixedStack) { runtime·printf("stackalloc: in malloc, size=%d want %d\n", FixedStack, n); runtime·throw("stackalloc"); } if(m->stackcachecnt == 0) stackcacherefill(); pos = m->stackcachepos; pos = (pos - 1) % StackCacheSize; v = m->stackcache[pos]; m->stackcachepos = pos; m->stackcachecnt--; m->stackinuse++; malloced = false; } else v = runtime·mallocgc(n, 0, FlagNoProfiling|FlagNoGC|FlagNoZero|FlagNoInvokeGC); top = (Stktop*)((byte*)v+n-sizeof(Stktop)); runtime·memclr((byte*)top, sizeof(*top)); top->malloced = malloced; return v; } void runtime·stackfree(G *gp, void *v, Stktop *top) { uint32 pos; uintptr n; n = (uintptr)(top+1) - (uintptr)v; if(StackDebug >= 1) runtime·printf("stackfree %p %d\n", v, (int32)n); gp->stacksize -= n; if(runtime·debug.efence || StackFromSystem) { if(runtime·debug.efence || StackFaultOnFree) runtime·SysFault(v, n); else runtime·SysFree(v, n, &mstats.stacks_sys); return; } if(top->malloced) { runtime·free(v); return; } if(n != FixedStack) runtime·throw("stackfree: bad fixed size"); if(m->stackcachecnt == StackCacheSize) stackcacherelease(); pos = m->stackcachepos; m->stackcache[pos] = v; m->stackcachepos = (pos + 1) % StackCacheSize; m->stackcachecnt++; m->stackinuse--; } // Called from runtime·lessstack when returning from a function which // allocated a new stack segment. The function's return value is in // m->cret. void runtime·oldstack(void) { Stktop *top; uint32 argsize; byte *sp, *old; uintptr *src, *dst, *dstend; G *gp; int64 goid; int32 oldstatus; gp = m->curg; top = (Stktop*)gp->stackbase; old = (byte*)gp->stackguard - StackGuard; sp = (byte*)top; argsize = top->argsize; if(StackDebug >= 1) { runtime·printf("runtime: oldstack gobuf={pc:%p sp:%p lr:%p} cret=%p argsize=%p\n", top->gobuf.pc, top->gobuf.sp, top->gobuf.lr, (uintptr)m->cret, (uintptr)argsize); } // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow // happens during a function call inside entersyscall. oldstatus = gp->status; gp->sched = top->gobuf; gp->sched.ret = m->cret; m->cret = 0; // drop reference gp->status = Gwaiting; gp->waitreason = "stack unsplit"; if(argsize > 0) { sp -= argsize; dst = (uintptr*)top->argp; dstend = dst + argsize/sizeof(*dst); src = (uintptr*)sp; while(dst < dstend) *dst++ = *src++; } goid = top->gobuf.g->goid; // fault if g is bad, before gogo USED(goid); gp->stackbase = top->stackbase; gp->stackguard = top->stackguard; gp->stackguard0 = gp->stackguard; gp->panicwrap = top->panicwrap; runtime·stackfree(gp, old, top); gp->status = oldstatus; runtime·gogo(&gp->sched); } uintptr runtime·maxstacksize = 1<<20; // enough until runtime.main sets it for real static uint8* mapnames[] = { (uint8*)"---", (uint8*)"scalar", (uint8*)"ptr", (uint8*)"multi", }; // Stack frame layout // // (x86) // +------------------+ // | args from caller | // +------------------+ <- frame->argp // | return address | // +------------------+ <- frame->varp // | locals | // +------------------+ // | args to callee | // +------------------+ <- frame->sp // // (arm: TODO) typedef struct CopyableInfo CopyableInfo; struct CopyableInfo { byte *stk; // bottom address of segment byte *base; // top address of segment (including Stktop) int32 frames; // count of copyable frames (-1 = not copyable) }; void runtime·main(void); static bool checkframecopy(Stkframe *frame, void *arg) { CopyableInfo *cinfo; Func *f; StackMap *stackmap; cinfo = arg; f = frame->fn; if(StackDebug >= 2) runtime·printf(" checking %s frame=[%p,%p] stk=[%p,%p]\n", runtime·funcname(f), frame->sp, frame->fp, cinfo->stk, cinfo->base); // if we're not in the segment any more, return immediately. if(frame->varp < cinfo->stk || frame->varp >= cinfo->base) { if(StackDebug >= 2) runtime·printf(" \n"); return false; // stop traceback } if(f->entry == (uintptr)runtime·main) { // A special routine at the TOS of the main routine. // We will allow it to be copied even though we don't // have full GC info for it (because it is written in C). cinfo->frames++; return false; // stop traceback } if(frame->varp != (byte*)frame->sp) { // not in prologue (and has at least one local or outarg) stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); if(stackmap == nil) { cinfo->frames = -1; if(StackDebug >= 1) runtime·printf("copystack: no locals info for %s\n", runtime·funcname(f)); return false; } if(stackmap->n <= 0) { cinfo->frames = -1; if(StackDebug >= 1) runtime·printf("copystack: locals size info only for %s\n", runtime·funcname(f)); return false; } } if(frame->arglen != 0) { stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); if(stackmap == nil) { cinfo->frames = -1; if(StackDebug >= 1) runtime·printf("copystack: no arg info for %s\n", runtime·funcname(f)); return false; } } cinfo->frames++; return true; // this frame is ok; keep going } // If the top segment of the stack contains an uncopyable // frame, return -1. Otherwise return the number of frames // in the top segment, all of which are copyable. static int32 copyabletopsegment(G *gp) { CopyableInfo cinfo; Defer *d; Func *f; FuncVal *fn; StackMap *stackmap; cinfo.stk = (byte*)gp->stackguard - StackGuard; cinfo.base = (byte*)gp->stackbase + sizeof(Stktop); cinfo.frames = 0; // Check that each frame is copyable. As a side effect, // count the frames. runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, checkframecopy, &cinfo, false); if(StackDebug >= 1 && cinfo.frames != -1) runtime·printf("copystack: %d copyable frames\n", cinfo.frames); // Check to make sure all Defers are copyable for(d = gp->defer; d != nil; d = d->link) { if(cinfo.stk <= (byte*)d && (byte*)d < cinfo.base) { // Defer is on the stack. Its copyableness has // been established during stack walking. // For now, this only happens with the Defer in runtime.main. continue; } if(d->argp < cinfo.stk || cinfo.base <= d->argp) break; // a defer for the next segment fn = d->fn; if(fn == nil) // See issue 8047 continue; f = runtime·findfunc((uintptr)fn->fn); if(f == nil) return -1; // Check to make sure we have an args pointer map for the defer's args. // We only need the args map, but we check // for the locals map also, because when the locals map // isn't provided it means the ptr map came from C and // C (particularly, cgo) lies to us. See issue 7695. stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); if(stackmap == nil || stackmap->n <= 0) return -1; stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); if(stackmap == nil || stackmap->n <= 0) return -1; if(cinfo.stk <= (byte*)fn && (byte*)fn < cinfo.base) { // FuncVal is on the stack. Again, its copyableness // was established during stack walking. continue; } // The FuncVal may have pointers in it, but fortunately for us // the compiler won't put pointers into the stack in a // heap-allocated FuncVal. // One day if we do need to check this, we'll need maps of the // pointerness of the closure args. The only place we have that map // right now is in the gc program for the FuncVal. Ugh. } return cinfo.frames; } typedef struct AdjustInfo AdjustInfo; struct AdjustInfo { byte *oldstk; // bottom address of segment byte *oldbase; // top address of segment (after Stktop) uintptr delta; // ptr distance from old to new stack (newbase - oldbase) }; // bv describes the memory starting at address scanp. // Adjust any pointers contained therein. static void adjustpointers(byte **scanp, BitVector *bv, AdjustInfo *adjinfo, Func *f) { uintptr delta; int32 num, i; byte *p, *minp, *maxp; Type *t; Itab *tab; minp = adjinfo->oldstk; maxp = adjinfo->oldbase; delta = adjinfo->delta; num = bv->n / BitsPerPointer; for(i = 0; i < num; i++) { if(StackDebug >= 4) runtime·printf(" %p:%s:%p\n", &scanp[i], mapnames[bv->data[i / (32 / BitsPerPointer)] >> (i * BitsPerPointer & 31) & 3], scanp[i]); switch(bv->data[i / (32 / BitsPerPointer)] >> (i * BitsPerPointer & 31) & 3) { case BitsDead: if(runtime·debug.gcdead) scanp[i] = (byte*)PoisonStack; break; case BitsScalar: break; case BitsPointer: p = scanp[i]; if(f != nil && (byte*)0 < p && (p < (byte*)PageSize || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) { // Looks like a junk value in a pointer slot. // Live analysis wrong? m->traceback = 2; runtime·printf("runtime: bad pointer in frame %s at %p: %p\n", runtime·funcname(f), &scanp[i], p); runtime·throw("bad pointer!"); } if(minp <= p && p < maxp) { if(StackDebug >= 3) runtime·printf("adjust ptr %p %s\n", p, runtime·funcname(f)); scanp[i] = p + delta; } break; case BitsMultiWord: switch(bv->data[(i+1) / (32 / BitsPerPointer)] >> ((i+1) * BitsPerPointer & 31) & 3) { case BitsString: // string referents are never on the stack, never need to be adjusted i++; // skip len break; case BitsSlice: p = scanp[i]; if(minp <= p && p < maxp) { if(StackDebug >= 3) runtime·printf("adjust slice %p\n", p); scanp[i] = p + delta; } i += 2; // skip len, cap break; case BitsEface: t = (Type*)scanp[i]; if(t != nil && (t->size > PtrSize || (t->kind & KindNoPointers) == 0)) { p = scanp[i+1]; if(minp <= p && p < maxp) { if(StackDebug >= 3) runtime·printf("adjust eface %p\n", p); if(t->size > PtrSize) // currently we always allocate such objects on the heap runtime·throw("large interface value found on stack"); scanp[i+1] = p + delta; } } i++; break; case BitsIface: tab = (Itab*)scanp[i]; if(tab != nil) { t = tab->type; //runtime·printf(" type=%p\n", t); if(t->size > PtrSize || (t->kind & KindNoPointers) == 0) { p = scanp[i+1]; if(minp <= p && p < maxp) { if(StackDebug >= 3) runtime·printf("adjust iface %p\n", p); if(t->size > PtrSize) // currently we always allocate such objects on the heap runtime·throw("large interface value found on stack"); scanp[i+1] = p + delta; } } } i++; break; } break; } } } // Note: the argument/return area is adjusted by the callee. static bool adjustframe(Stkframe *frame, void *arg) { AdjustInfo *adjinfo; Func *f; StackMap *stackmap; int32 pcdata; BitVector bv; uintptr targetpc; adjinfo = arg; f = frame->fn; if(StackDebug >= 2) runtime·printf(" adjusting %s frame=[%p,%p] pc=%p continpc=%p\n", runtime·funcname(f), frame->sp, frame->fp, frame->pc, frame->continpc); if(f->entry == (uintptr)runtime·main) return true; targetpc = frame->continpc; if(targetpc == 0) { // Frame is dead. return true; } if(targetpc != f->entry) targetpc--; pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc); if(pcdata == -1) pcdata = 0; // in prologue // adjust local pointers if(frame->varp != (byte*)frame->sp) { stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps); if(stackmap == nil) runtime·throw("no locals info"); if(stackmap->n <= 0) runtime·throw("locals size info only"); bv = runtime·stackmapdata(stackmap, pcdata); if(StackDebug >= 3) runtime·printf(" locals\n"); adjustpointers((byte**)frame->varp - bv.n / BitsPerPointer, &bv, adjinfo, f); } // adjust inargs and outargs if(frame->arglen != 0) { stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); if(stackmap == nil) runtime·throw("no arg info"); bv = runtime·stackmapdata(stackmap, pcdata); if(StackDebug >= 3) runtime·printf(" args\n"); adjustpointers((byte**)frame->argp, &bv, adjinfo, nil); } return true; } static void adjustctxt(G *gp, AdjustInfo *adjinfo) { if(adjinfo->oldstk <= (byte*)gp->sched.ctxt && (byte*)gp->sched.ctxt < adjinfo->oldbase) gp->sched.ctxt = (byte*)gp->sched.ctxt + adjinfo->delta; } static void adjustdefers(G *gp, AdjustInfo *adjinfo) { Defer *d, **dp; Func *f; FuncVal *fn; StackMap *stackmap; BitVector bv; for(dp = &gp->defer, d = *dp; d != nil; dp = &d->link, d = *dp) { if(adjinfo->oldstk <= (byte*)d && (byte*)d < adjinfo->oldbase) { // The Defer record is on the stack. Its fields will // get adjusted appropriately. // This only happens for runtime.main now, but a compiler // optimization could do more of this. *dp = (Defer*)((byte*)d + adjinfo->delta); continue; } if(d->argp < adjinfo->oldstk || adjinfo->oldbase <= d->argp) break; // a defer for the next segment fn = d->fn; if(fn == nil) { // Defer of nil function. It will panic when run, and there // aren't any args to adjust. See issue 8047. d->argp += adjinfo->delta; continue; } f = runtime·findfunc((uintptr)fn->fn); if(f == nil) runtime·throw("can't adjust unknown defer"); if(StackDebug >= 4) runtime·printf(" checking defer %s\n", runtime·funcname(f)); // Defer's FuncVal might be on the stack if(adjinfo->oldstk <= (byte*)fn && (byte*)fn < adjinfo->oldbase) { if(StackDebug >= 3) runtime·printf(" adjust defer fn %s\n", runtime·funcname(f)); d->fn = (FuncVal*)((byte*)fn + adjinfo->delta); } else { // deferred function's args might point into the stack. if(StackDebug >= 3) runtime·printf(" adjust deferred args for %s\n", runtime·funcname(f)); stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps); if(stackmap == nil) runtime·throw("runtime: deferred function has no arg ptr map"); bv = runtime·stackmapdata(stackmap, 0); adjustpointers(d->args, &bv, adjinfo, f); } d->argp += adjinfo->delta; } } // Copies the top stack segment of gp to a new stack segment of a // different size. The top segment must contain nframes frames. static void copystack(G *gp, uintptr nframes, uintptr newsize) { byte *oldstk, *oldbase, *newstk, *newbase; uintptr oldsize, used; AdjustInfo adjinfo; Stktop *oldtop, *newtop; bool malloced; if(gp->syscallstack != 0) runtime·throw("can't handle stack copy in syscall yet"); oldstk = (byte*)gp->stackguard - StackGuard; oldbase = (byte*)gp->stackbase + sizeof(Stktop); oldsize = oldbase - oldstk; used = oldbase - (byte*)gp->sched.sp; oldtop = (Stktop*)gp->stackbase; // allocate new stack newstk = runtime·stackalloc(gp, newsize); newbase = newstk + newsize; newtop = (Stktop*)(newbase - sizeof(Stktop)); malloced = newtop->malloced; if(StackDebug >= 1) runtime·printf("copystack [%p %p]/%d -> [%p %p]/%d\n", oldstk, oldbase, (int32)oldsize, newstk, newbase, (int32)newsize); USED(oldsize); // adjust pointers in the to-be-copied frames adjinfo.oldstk = oldstk; adjinfo.oldbase = oldbase; adjinfo.delta = newbase - oldbase; runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, nframes, adjustframe, &adjinfo, false); // adjust other miscellaneous things that have pointers into stacks. adjustctxt(gp, &adjinfo); adjustdefers(gp, &adjinfo); // copy the stack (including Stktop) to the new location runtime·memmove(newbase - used, oldbase - used, used); newtop->malloced = malloced; // Swap out old stack for new one gp->stackbase = (uintptr)newtop; gp->stackguard = (uintptr)newstk + StackGuard; gp->stackguard0 = (uintptr)newstk + StackGuard; // NOTE: might clobber a preempt request if(gp->stack0 == (uintptr)oldstk) gp->stack0 = (uintptr)newstk; gp->sched.sp = (uintptr)(newbase - used); // free old stack runtime·stackfree(gp, oldstk, oldtop); } // round x up to a power of 2. int32 runtime·round2(int32 x) { int32 s; s = 0; while((1 << s) < x) s++; return 1 << s; } // Called from runtime·newstackcall or from runtime·morestack when a new // stack segment is needed. Allocate a new stack big enough for // m->moreframesize bytes, copy m->moreargsize bytes to the new frame, // and then act as though runtime·lessstack called the function at // m->morepc. void runtime·newstack(void) { int32 framesize, argsize, oldstatus, oldsize, newsize, nframes; Stktop *top, *oldtop; byte *stk, *oldstk, *oldbase; uintptr sp; uintptr *src, *dst, *dstend; G *gp; Gobuf label, morebuf; void *moreargp; bool newstackcall; if(m->forkstackguard) runtime·throw("split stack after fork"); if(m->morebuf.g != m->curg) { runtime·printf("runtime: newstack called from g=%p\n" "\tm=%p m->curg=%p m->g0=%p m->gsignal=%p\n", m->morebuf.g, m, m->curg, m->g0, m->gsignal); runtime·throw("runtime: wrong goroutine in newstack"); } // gp->status is usually Grunning, but it could be Gsyscall if a stack overflow // happens during a function call inside entersyscall. gp = m->curg; oldstatus = gp->status; framesize = m->moreframesize; argsize = m->moreargsize; moreargp = m->moreargp; m->moreargp = nil; morebuf = m->morebuf; m->morebuf.pc = (uintptr)nil; m->morebuf.lr = (uintptr)nil; m->morebuf.sp = (uintptr)nil; gp->status = Gwaiting; gp->waitreason = "stack growth"; newstackcall = framesize==1; if(newstackcall) framesize = 0; // For newstackcall the context already points to beginning of runtime·newstackcall. if(!newstackcall) runtime·rewindmorestack(&gp->sched); sp = gp->sched.sp; if(thechar == '6' || thechar == '8') { // The call to morestack cost a word. sp -= sizeof(uintptr); } if(StackDebug >= 1 || sp < gp->stackguard - StackGuard) { runtime·printf("runtime: newstack framesize=%p argsize=%p sp=%p stack=[%p, %p]\n" "\tmorebuf={pc:%p sp:%p lr:%p}\n" "\tsched={pc:%p sp:%p lr:%p ctxt:%p}\n", (uintptr)framesize, (uintptr)argsize, sp, gp->stackguard - StackGuard, gp->stackbase, m->morebuf.pc, m->morebuf.sp, m->morebuf.lr, gp->sched.pc, gp->sched.sp, gp->sched.lr, gp->sched.ctxt); } if(sp < gp->stackguard - StackGuard) { runtime·printf("runtime: split stack overflow: %p < %p\n", sp, gp->stackguard - StackGuard); runtime·throw("runtime: split stack overflow"); } if(argsize % sizeof(uintptr) != 0) { runtime·printf("runtime: stack growth with misaligned argsize %d\n", argsize); runtime·throw("runtime: stack growth argsize"); } if(gp->stackguard0 == (uintptr)StackPreempt) { if(gp == m->g0) runtime·throw("runtime: preempt g0"); if(oldstatus == Grunning && m->p == nil && m->locks == 0) runtime·throw("runtime: g is running but p is not"); if(oldstatus == Gsyscall && m->locks == 0) runtime·throw("runtime: stack growth during syscall"); // Be conservative about where we preempt. // We are interested in preempting user Go code, not runtime code. if(oldstatus != Grunning || m->locks || m->mallocing || m->gcing || m->p->status != Prunning) { // Let the goroutine keep running for now. // gp->preempt is set, so it will be preempted next time. gp->stackguard0 = gp->stackguard; gp->status = oldstatus; runtime·gogo(&gp->sched); // never return } // Act like goroutine called runtime.Gosched. gp->status = oldstatus; runtime·gosched0(gp); // never return } // If every frame on the top segment is copyable, allocate a bigger segment // and move the segment instead of allocating a new segment. if(runtime·copystack) { if(!runtime·precisestack) runtime·throw("can't copy stacks without precise stacks"); nframes = copyabletopsegment(gp); if(nframes != -1) { oldstk = (byte*)gp->stackguard - StackGuard; oldbase = (byte*)gp->stackbase + sizeof(Stktop); oldsize = oldbase - oldstk; newsize = oldsize * 2; copystack(gp, nframes, newsize); if(StackDebug >= 1) runtime·printf("stack grow done\n"); if(gp->stacksize > runtime·maxstacksize) { runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); runtime·throw("stack overflow"); } gp->status = oldstatus; runtime·gogo(&gp->sched); } // TODO: if stack is uncopyable because we're in C code, patch return value at // end of C code to trigger a copy as soon as C code exits. That way, we'll // have stack available if we get this deep again. } // allocate new segment. framesize += argsize; framesize += StackExtra; // room for more functions, Stktop. if(framesize < StackMin) framesize = StackMin; framesize += StackSystem; framesize = runtime·round2(framesize); stk = runtime·stackalloc(gp, framesize); if(gp->stacksize > runtime·maxstacksize) { runtime·printf("runtime: goroutine stack exceeds %D-byte limit\n", (uint64)runtime·maxstacksize); runtime·throw("stack overflow"); } top = (Stktop*)(stk+framesize-sizeof(*top)); if(StackDebug >= 1) { runtime·printf("\t-> new stack [%p, %p]\n", stk, top); } top->stackbase = gp->stackbase; top->stackguard = gp->stackguard; top->gobuf = morebuf; top->argp = moreargp; top->argsize = argsize; // copy flag from panic top->panic = gp->ispanic; gp->ispanic = false; // if this isn't a panic, maybe we're splitting the stack for a panic. // if we're splitting in the top frame, propagate the panic flag // forward so that recover will know we're in a panic. oldtop = (Stktop*)top->stackbase; if(oldtop != nil && oldtop->panic && top->argp == (byte*)oldtop - oldtop->argsize - gp->panicwrap) top->panic = true; top->panicwrap = gp->panicwrap; gp->panicwrap = 0; gp->stackbase = (uintptr)top; gp->stackguard = (uintptr)stk + StackGuard; gp->stackguard0 = gp->stackguard; sp = (uintptr)top; if(argsize > 0) { sp -= argsize; dst = (uintptr*)sp; dstend = dst + argsize/sizeof(*dst); src = (uintptr*)top->argp; while(dst < dstend) *dst++ = *src++; } if(thechar == '5') { // caller would have saved its LR below args. sp -= sizeof(void*); *(void**)sp = nil; } // Continue as if lessstack had just called m->morepc // (the PC that decided to grow the stack). runtime·memclr((byte*)&label, sizeof label); label.sp = sp; label.pc = (uintptr)runtime·lessstack; label.g = m->curg; if(newstackcall) runtime·gostartcallfn(&label, (FuncVal*)m->cret); else { runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt); gp->sched.ctxt = nil; } gp->status = oldstatus; runtime·gogo(&label); *(int32*)345 = 123; // never return } #pragma textflag NOSPLIT void runtime·nilfunc(void) { *(byte*)0 = 0; } // adjust Gobuf as if it executed a call to fn // and then did an immediate gosave. void runtime·gostartcallfn(Gobuf *gobuf, FuncVal *fv) { void *fn; if(fv != nil) fn = fv->fn; else fn = runtime·nilfunc; runtime·gostartcall(gobuf, fn, fv); } // Maybe shrink the stack being used by gp. // Called at garbage collection time. void runtime·shrinkstack(G *gp) { int32 nframes; byte *oldstk, *oldbase; uintptr used, oldsize, newsize; MSpan *span; if(!runtime·copystack) return; oldstk = (byte*)gp->stackguard - StackGuard; oldbase = (byte*)gp->stackbase + sizeof(Stktop); oldsize = oldbase - oldstk; newsize = oldsize / 2; if(newsize < FixedStack) return; // don't shrink below the minimum-sized stack used = oldbase - (byte*)gp->sched.sp; if(used >= oldsize / 4) return; // still using at least 1/4 of the segment. // To shrink to less than 1/2 a page, we need to copy. if(newsize < PageSize/2) { if(gp->syscallstack != (uintptr)nil) // TODO: can we handle this case? return; #ifdef GOOS_windows if(gp->m != nil && gp->m->libcallsp != 0) return; #endif nframes = copyabletopsegment(gp); if(nframes == -1) return; copystack(gp, nframes, newsize); return; } // To shrink a stack of one page size or more, we can shrink it // without copying. Just deallocate the lower half. span = runtime·MHeap_LookupMaybe(&runtime·mheap, oldstk); if(span == nil) return; // stack allocated outside heap. Can't shrink it. Can happen if stack is allocated while inside malloc. TODO: shrink by copying? if(span->elemsize != oldsize) runtime·throw("span element size doesn't match stack size"); if((uintptr)oldstk != span->start << PageShift) runtime·throw("stack not at start of span"); if(StackDebug) runtime·printf("shrinking stack in place %p %X->%X\n", oldstk, oldsize, newsize); // new stack guard for smaller stack gp->stackguard = (uintptr)oldstk + newsize + StackGuard; gp->stackguard0 = (uintptr)oldstk + newsize + StackGuard; if(gp->stack0 == (uintptr)oldstk) gp->stack0 = (uintptr)oldstk + newsize; gp->stacksize -= oldsize - newsize; // Free bottom half of the stack. if(runtime·debug.efence || StackFromSystem) { if(runtime·debug.efence || StackFaultOnFree) runtime·SysFault(oldstk, newsize); else runtime·SysFree(oldstk, newsize, &mstats.stacks_sys); return; } // First, we trick malloc into thinking // we allocated the stack as two separate half-size allocs. Then the // free() call does the rest of the work for us. runtime·MSpan_EnsureSwept(span); runtime·MHeap_SplitSpan(&runtime·mheap, span); runtime·free(oldstk); }