diff options
Diffstat (limited to 'src/pkg/runtime')
87 files changed, 1366 insertions, 2044 deletions
diff --git a/src/pkg/runtime/386/asm.s b/src/pkg/runtime/386/asm.s index 84f5367e5..63d582606 100644 --- a/src/pkg/runtime/386/asm.s +++ b/src/pkg/runtime/386/asm.s @@ -18,9 +18,10 @@ TEXT _rt0_386(SB),7,$0 // we set up GS ourselves. MOVL initcgo(SB), AX TESTL AX, AX - JZ 3(PC) + JZ 4(PC) CALL AX - JMP ok + CMPL runtime·iswindows(SB), $0 + JEQ ok // set up %gs CALL runtime·ldt0setup(SB) @@ -46,7 +47,7 @@ ok: MOVL CX, m_g0(AX) // create istack out of the OS stack - LEAL (-8192+104)(SP), AX // TODO: 104? + LEAL (-64*1024+104)(SP), AX // TODO: 104? MOVL AX, g_stackguard(CX) MOVL SP, g_stackbase(CX) CALL runtime·emptyfunc(SB) // fault if stack check is wrong @@ -156,8 +157,8 @@ TEXT runtime·morestack(SB),7,$0 // frame size in DX // arg size in AX // Save in m. - MOVL DX, m_moreframe(BX) - MOVL AX, m_moreargs(BX) + MOVL DX, m_moreframesize(BX) + MOVL AX, m_moreargsize(BX) // Called from f. // Set m->morebuf to f's caller. @@ -165,7 +166,7 @@ TEXT runtime·morestack(SB),7,$0 MOVL DI, (m_morebuf+gobuf_pc)(BX) LEAL 8(SP), CX // f's caller's SP MOVL CX, (m_morebuf+gobuf_sp)(BX) - MOVL CX, (m_morefp)(BX) + MOVL CX, m_moreargp(BX) get_tls(CX) MOVL g(CX), SI MOVL SI, (m_morebuf+gobuf_g)(BX) @@ -213,9 +214,9 @@ TEXT reflect·call(SB), 7, $0 MOVL 12(SP), CX // arg size MOVL AX, m_morepc(BX) // f's PC - MOVL DX, m_morefp(BX) // argument frame pointer - MOVL CX, m_moreargs(BX) // f's argument size - MOVL $1, m_moreframe(BX) // f's frame size + MOVL DX, m_moreargp(BX) // f's argument pointer + MOVL CX, m_moreargsize(BX) // f's argument size + MOVL $1, m_moreframesize(BX) // f's frame size // Call newstack on m's scheduling stack. MOVL m_g0(BX), BP diff --git a/src/pkg/runtime/Makefile b/src/pkg/runtime/Makefile index e62dbe393..e9488cfb5 100644 --- a/src/pkg/runtime/Makefile +++ b/src/pkg/runtime/Makefile @@ -26,16 +26,7 @@ GOFILES=\ softfloat64.go\ type.go\ version.go\ - chan_defs.go\ - hashmap_defs.go\ - iface_defs.go\ - malloc_defs.go\ - mheapmap$(SIZE)_defs.go\ runtime_defs.go\ - $(GOOS)/runtime_defs.go\ - -GOFILES_tiny=\ - tiny/io.go\ OFILES_windows=\ syscall.$O\ @@ -73,7 +64,6 @@ OFILES=\ mfixalloc.$O\ mgc0.$O\ mheap.$O\ - mheapmap$(SIZE).$O\ mprof.$O\ msize.$O\ print.$O\ @@ -122,7 +112,7 @@ $(pkgdir)/%.h: %.h clean: clean-local clean-local: - rm -f goc2c mkversion version.go */asm.h runtime.acid.* $$(ls *.goc | sed 's/goc$$/c/') + rm -f goc2c mkversion version.go */asm.h runtime.acid.* runtime_defs.go $$(ls *.goc | sed 's/goc$$/c/') $(GOARCH)/asm.h: mkasmh.sh runtime.acid.$(GOARCH) ./mkasmh.sh >$@.x @@ -165,3 +155,7 @@ ifeq ($(GOARCH),386) traceback.$O: amd64/traceback.c $(CC) $(CFLAGS) $< endif + +runtime_defs.go: proc.c iface.c hashmap.c chan.c + CC="$(CC)" CFLAGS="$(CFLAGS)" ./mkgodefs.sh $^ > $@.x + mv -f $@.x $@ diff --git a/src/pkg/runtime/amd64/asm.s b/src/pkg/runtime/amd64/asm.s index 235f27206..b6642c13c 100644 --- a/src/pkg/runtime/amd64/asm.s +++ b/src/pkg/runtime/amd64/asm.s @@ -151,7 +151,7 @@ TEXT runtime·morestack(SB),7,$0 MOVQ AX, (m_morebuf+gobuf_pc)(BX) LEAQ 16(SP), AX // f's caller's SP MOVQ AX, (m_morebuf+gobuf_sp)(BX) - MOVQ AX, (m_morefp)(BX) + MOVQ AX, m_moreargp(BX) get_tls(CX) MOVQ g(CX), SI MOVQ SI, (m_morebuf+gobuf_g)(BX) @@ -197,9 +197,9 @@ TEXT reflect·call(SB), 7, $0 MOVL 24(SP), CX // arg size MOVQ AX, m_morepc(BX) // f's PC - MOVQ DX, m_morefp(BX) // argument frame pointer - MOVL CX, m_moreargs(BX) // f's argument size - MOVL $1, m_moreframe(BX) // f's frame size + MOVQ DX, m_moreargp(BX) // argument frame pointer + MOVL CX, m_moreargsize(BX) // f's argument size + MOVL $1, m_moreframesize(BX) // f's frame size // Call newstack on m's scheduling stack. MOVQ m_g0(BX), BP @@ -230,7 +230,7 @@ TEXT runtime·morestack00(SB),7,$0 get_tls(CX) MOVQ m(CX), BX MOVQ $0, AX - MOVQ AX, m_moreframe(BX) + MOVQ AX, m_moreframesize(BX) MOVQ $runtime·morestack(SB), AX JMP AX @@ -238,7 +238,7 @@ TEXT runtime·morestack01(SB),7,$0 get_tls(CX) MOVQ m(CX), BX SHLQ $32, AX - MOVQ AX, m_moreframe(BX) + MOVQ AX, m_moreframesize(BX) MOVQ $runtime·morestack(SB), AX JMP AX @@ -246,14 +246,14 @@ TEXT runtime·morestack10(SB),7,$0 get_tls(CX) MOVQ m(CX), BX MOVLQZX AX, AX - MOVQ AX, m_moreframe(BX) + MOVQ AX, m_moreframesize(BX) MOVQ $runtime·morestack(SB), AX JMP AX TEXT runtime·morestack11(SB),7,$0 get_tls(CX) MOVQ m(CX), BX - MOVQ AX, m_moreframe(BX) + MOVQ AX, m_moreframesize(BX) MOVQ $runtime·morestack(SB), AX JMP AX @@ -294,7 +294,7 @@ TEXT morestack<>(SB),7,$0 MOVQ m(CX), BX POPQ AX SHLQ $35, AX - MOVQ AX, m_moreframe(BX) + MOVQ AX, m_moreframesize(BX) MOVQ $runtime·morestack(SB), AX JMP AX @@ -407,9 +407,9 @@ TEXT runtime·stackcheck(SB), 7, $0 TEXT runtime·memclr(SB),7,$0 MOVQ 8(SP), DI // arg 1 addr - MOVL 16(SP), CX // arg 2 count - ADDL $7, CX - SHRL $3, CX + MOVQ 16(SP), CX // arg 2 count + ADDQ $7, CX + SHRQ $3, CX MOVQ $0, AX CLD REP diff --git a/src/pkg/runtime/amd64/traceback.c b/src/pkg/runtime/amd64/traceback.c index 3ea80a661..86e96f348 100644 --- a/src/pkg/runtime/amd64/traceback.c +++ b/src/pkg/runtime/amd64/traceback.c @@ -60,7 +60,7 @@ gentraceback(byte *pc0, byte *sp, G *g, int32 skip, uintptr *pcbuf, int32 m) // The 0x48 byte is only on amd64. p = (byte*)pc; // We check p < p+8 to avoid wrapping and faulting if we lose track. - if(runtime·mheap.min < p && p < p+8 && p+8 < runtime·mheap.max && // pointer in allocated memory + if(runtime·mheap.arena_start < p && p < p+8 && p+8 < runtime·mheap.arena_used && // pointer in allocated memory (sizeof(uintptr) != 8 || *p++ == 0x48) && // skip 0x48 byte on amd64 p[0] == 0x81 && p[1] == 0xc4 && p[6] == 0xc3) { sp += *(uint32*)(p+2); @@ -154,7 +154,7 @@ isclosureentry(uintptr pc) int32 i, siz; p = (byte*)pc; - if(p < runtime·mheap.min || p+32 > runtime·mheap.max) + if(p < runtime·mheap.arena_start || p+32 > runtime·mheap.arena_used) return 0; // SUBQ $siz, SP diff --git a/src/pkg/runtime/arm/asm.s b/src/pkg/runtime/arm/asm.s index 44c47bad1..a4e4b3283 100644 --- a/src/pkg/runtime/arm/asm.s +++ b/src/pkg/runtime/arm/asm.s @@ -145,14 +145,15 @@ TEXT runtime·morestack(SB),7,$-4 BL.EQ runtime·abort(SB) // Save in m. - MOVW R1, m_moreframe(m) - MOVW R2, m_moreargs(m) + MOVW R1, m_moreframesize(m) + MOVW R2, m_moreargsize(m) // Called from f. // Set m->morebuf to f's caller. MOVW R3, (m_morebuf+gobuf_pc)(m) // f's caller's PC MOVW SP, (m_morebuf+gobuf_sp)(m) // f's caller's SP - MOVW SP, m_morefp(m) // f's caller's SP + MOVW $4(SP), R3 // f's argument pointer + MOVW R3, m_moreargp(m) MOVW g, (m_morebuf+gobuf_g)(m) // Set m->morepc to f's PC. @@ -185,14 +186,11 @@ TEXT reflect·call(SB), 7, $-4 MOVW 8(SP), R1 // arg frame MOVW 12(SP), R2 // arg size - SUB $4,R1 // add the saved LR to the frame - ADD $4,R2 - MOVW R0, m_morepc(m) // f's PC - MOVW R1, m_morefp(m) // argument frame pointer - MOVW R2, m_moreargs(m) // f's argument size + MOVW R1, m_moreargp(m) // f's argument pointer + MOVW R2, m_moreargsize(m) // f's argument size MOVW $1, R3 - MOVW R3, m_moreframe(m) // f's frame size + MOVW R3, m_moreframesize(m) // f's frame size // Call newstack on m's scheduling stack. MOVW m_g0(m), g @@ -218,8 +216,9 @@ TEXT runtime·lessstack(SB), 7, $-4 TEXT runtime·jmpdefer(SB), 7, $0 MOVW 0(SP), LR MOVW $-4(LR), LR // BL deferreturn - MOVW 4(SP), R0 // fn - MOVW 8(SP), SP + MOVW fn+0(FP), R0 + MOVW argp+4(FP), SP + MOVW $-4(SP), SP // SP is 4 below argp, due to saved LR B (R0) TEXT runtime·memclr(SB),7,$20 diff --git a/src/pkg/runtime/cgo/Makefile b/src/pkg/runtime/cgo/Makefile index 55b6967d9..768fe80ac 100644 --- a/src/pkg/runtime/cgo/Makefile +++ b/src/pkg/runtime/cgo/Makefile @@ -10,6 +10,10 @@ ifeq ($(GOARCH),arm) ENABLED:=0 endif +ifeq ($(DISABLE_CGO),1) +ENABLED:=0 +endif + TARG=runtime/cgo GOFILES=\ @@ -30,7 +34,11 @@ OFILES=\ _cgo_import.$O\ $(CGO_OFILES)\ +ifeq ($(GOOS),windows) +CGO_LDFLAGS=-lm -mthreads +else CGO_LDFLAGS=-lpthread +endif ifeq ($(GOOS),freebsd) OFILES+=\ @@ -48,7 +56,6 @@ _cgo_defun.c: _cgo_main.c: echo 'int main() { return 0; }' >$@ - echo 'void *crosscall2;' >>$@ endif $(GOARCH).o: $(GOARCH).S diff --git a/src/pkg/runtime/cgo/windows_386.c b/src/pkg/runtime/cgo/windows_386.c index 5f5235bd2..f39309cb1 100755 --- a/src/pkg/runtime/cgo/windows_386.c +++ b/src/pkg/runtime/cgo/windows_386.c @@ -30,6 +30,7 @@ static void* threadentry(void *v) { ThreadStart ts; + void *tls0; ts = *(ThreadStart*)v; free(v); @@ -45,13 +46,17 @@ threadentry(void *v) /* * Set specific keys in thread local storage. */ + tls0 = (void*)LocalAlloc(LPTR, 32); asm volatile ( - "MOVL %%fs:0x2c, %%eax\n" // MOVL 0x24(FS), tmp - "movl %0, 0(%%eax)\n" // MOVL g, 0(FS) - "movl %1, 4(%%eax)\n" // MOVL m, 4(FS) - :: "r"(ts.g), "r"(ts.m) : "%eax" + "movl %0, %%fs:0x2c\n" // MOVL tls0, 0x2c(FS) + "movl %%fs:0x2c, %%eax\n" // MOVL 0x2c(FS), tmp + "movl %1, 0(%%eax)\n" // MOVL g, 0(FS) + "movl %2, 4(%%eax)\n" // MOVL m, 4(FS) + :: "r"(tls0), "r"(ts.g), "r"(ts.m) : "%eax" ); crosscall_386(ts.fn); + + LocalFree(tls0); return nil; } diff --git a/src/pkg/runtime/cgocall.c b/src/pkg/runtime/cgocall.c index 80ae97e7a..e6ece9542 100644 --- a/src/pkg/runtime/cgocall.c +++ b/src/pkg/runtime/cgocall.c @@ -53,12 +53,13 @@ runtime·cgocall(void (*fn)(void*), void *arg) // (arg/argsize) on to the stack, calls the function, copies the // arguments back where they came from, and finally returns to the old // stack. -void +uintptr runtime·cgocallback(void (*fn)(void), void *arg, int32 argsize) { Gobuf oldsched, oldg1sched; G *g1; void *sp; + uintptr ret; if(g != m->g0) runtime·throw("bad g in cgocallback"); @@ -70,11 +71,11 @@ runtime·cgocallback(void (*fn)(void), void *arg, int32 argsize) runtime·startcgocallback(g1); sp = g1->sched.sp - argsize; - if(sp < g1->stackguard) + if(sp < g1->stackguard - StackGuard + 4) // +4 for return address runtime·throw("g stack overflow in cgocallback"); runtime·mcpy(sp, arg, argsize); - runtime·runcgocallback(g1, sp, fn); + ret = runtime·runcgocallback(g1, sp, fn); runtime·mcpy(arg, sp, argsize); @@ -82,6 +83,8 @@ runtime·cgocallback(void (*fn)(void), void *arg, int32 argsize) m->sched = oldsched; g1->sched = oldg1sched; + + return ret; } void diff --git a/src/pkg/runtime/cgocall.h b/src/pkg/runtime/cgocall.h index 1ad954eb1..7c24e167b 100644 --- a/src/pkg/runtime/cgocall.h +++ b/src/pkg/runtime/cgocall.h @@ -7,6 +7,6 @@ */ void runtime·cgocall(void (*fn)(void*), void*); -void runtime·cgocallback(void (*fn)(void), void*, int32); +uintptr runtime·cgocallback(void (*fn)(void), void*, int32); void *runtime·cmalloc(uintptr); void runtime·cfree(void*); diff --git a/src/pkg/runtime/chan.c b/src/pkg/runtime/chan.c index 94ea513e7..8d3ac2ca4 100644 --- a/src/pkg/runtime/chan.c +++ b/src/pkg/runtime/chan.c @@ -11,8 +11,6 @@ enum { Wclosed = 0x0001, // writer has closed Rclosed = 0x0002, // reader has seen close - Eincr = 0x0004, // increment errors - Emax = 0x0800, // error limit before throw }; typedef struct Link Link; @@ -76,6 +74,7 @@ struct Select uint16 tcase; // total count of scase[] uint16 ncase; // currently filled scase[] Select* link; // for freelist + uint16* order; Scase* scase[1]; // one per case }; @@ -84,9 +83,7 @@ static SudoG* dequeue(WaitQ*, Hchan*); static void enqueue(WaitQ*, SudoG*); static SudoG* allocsg(Hchan*); static void freesg(Hchan*, SudoG*); -static uint32 gcd(uint32, uint32); -static uint32 fastrand1(void); -static uint32 fastrand2(void); +static uint32 fastrandn(uint32); static void destroychan(Hchan*); Hchan* @@ -152,16 +149,6 @@ runtime·makechan(Type *elem, int64 hint, Hchan *ret) FLUSH(&ret); } -static void -incerr(Hchan* c) -{ - c->closed += Eincr; - if(c->closed & Emax) { - // Note that channel locks may still be held at this point. - runtime·throw("too many operations on a closed channel"); - } -} - /* * generic single channel send/recv * if the bool pointer is nil, @@ -277,14 +264,12 @@ asynch: return; closed: - incerr(c); - if(pres != nil) - *pres = true; runtime·unlock(c); + runtime·panicstring("send on closed channel"); } void -runtime·chanrecv(Hchan* c, byte *ep, bool* pres) +runtime·chanrecv(Hchan* c, byte *ep, bool *pres, bool *closed) { SudoG *sg; G *gp; @@ -299,6 +284,9 @@ runtime·chanrecv(Hchan* c, byte *ep, bool* pres) runtime·printf("chanrecv: chan=%p\n", c); runtime·lock(c); + if(closed != nil) + *closed = false; + loop: if(c->dataqsiz > 0) goto asynch; @@ -308,7 +296,8 @@ loop: sg = dequeue(&c->sendq, c); if(sg != nil) { - c->elemalg->copy(c->elemsize, ep, sg->elem); + if(ep != nil) + c->elemalg->copy(c->elemsize, ep, sg->elem); c->elemalg->copy(c->elemsize, sg->elem, nil); gp = sg->g; @@ -323,7 +312,6 @@ loop: if(pres != nil) { runtime·unlock(c); - c->elemalg->copy(c->elemsize, ep, nil); *pres = false; return; } @@ -340,7 +328,8 @@ loop: if(sg == nil) goto loop; - c->elemalg->copy(c->elemsize, ep, sg->elem); + if(ep != nil) + c->elemalg->copy(c->elemsize, ep, sg->elem); c->elemalg->copy(c->elemsize, sg->elem, nil); freesg(c, sg); runtime·unlock(c); @@ -353,7 +342,6 @@ asynch: if(pres != nil) { runtime·unlock(c); - c->elemalg->copy(c->elemsize, ep, nil); *pres = false; return; } @@ -366,7 +354,8 @@ asynch: runtime·lock(c); goto asynch; } - c->elemalg->copy(c->elemsize, ep, c->recvdataq->elem); + if(ep != nil) + c->elemalg->copy(c->elemsize, ep, c->recvdataq->elem); c->elemalg->copy(c->elemsize, c->recvdataq->elem, nil); c->recvdataq = c->recvdataq->link; c->qcount--; @@ -387,9 +376,11 @@ asynch: return; closed: - c->elemalg->copy(c->elemsize, ep, nil); + if(closed != nil) + *closed = true; + if(ep != nil) + c->elemalg->copy(c->elemsize, ep, nil); c->closed |= Rclosed; - incerr(c); if(pres != nil) *pres = true; runtime·unlock(c); @@ -411,55 +402,99 @@ runtime·chansend1(Hchan* c, ...) runtime·chansend(c, ae, nil); } -// chansend2(hchan *chan any, elem any) (pres bool); +// chanrecv1(hchan *chan any) (elem any); #pragma textflag 7 void -runtime·chansend2(Hchan* c, ...) +runtime·chanrecv1(Hchan* c, ...) { int32 o; - byte *ae, *ap; - - if(c == nil) - runtime·panicstring("send to nil channel"); + byte *ae; - o = runtime·rnd(sizeof(c), c->elemalign); + o = runtime·rnd(sizeof(c), Structrnd); ae = (byte*)&c + o; - o = runtime·rnd(o+c->elemsize, Structrnd); - ap = (byte*)&c + o; - runtime·chansend(c, ae, ap); + runtime·chanrecv(c, ae, nil, nil); } -// chanrecv1(hchan *chan any) (elem any); +// chanrecv3(hchan *chan any) (elem any, closed bool); #pragma textflag 7 void -runtime·chanrecv1(Hchan* c, ...) +runtime·chanrecv3(Hchan* c, ...) { int32 o; - byte *ae; + byte *ae, *ac; + + if(c == nil) + runtime·panicstring("range over nil channel"); o = runtime·rnd(sizeof(c), Structrnd); ae = (byte*)&c + o; + o = runtime·rnd(o+c->elemsize, 1); + ac = (byte*)&c + o; - runtime·chanrecv(c, ae, nil); + runtime·chanrecv(c, ae, nil, ac); } -// chanrecv2(hchan *chan any) (elem any, pres bool); +// func selectnbsend(c chan any, elem any) bool +// +// compiler implements +// +// select { +// case c <- v: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if c != nil && selectnbsend(c, v) { +// ... foo +// } else { +// ... bar +// } +// #pragma textflag 7 void -runtime·chanrecv2(Hchan* c, ...) +runtime·selectnbsend(Hchan *c, ...) { int32 o; byte *ae, *ap; - o = runtime·rnd(sizeof(c), Structrnd); + o = runtime·rnd(sizeof(c), c->elemalign); ae = (byte*)&c + o; - o = runtime·rnd(o+c->elemsize, 1); + o = runtime·rnd(o+c->elemsize, Structrnd); ap = (byte*)&c + o; - runtime·chanrecv(c, ae, ap); + runtime·chansend(c, ae, ap); } +// func selectnbrecv(elem *any, c chan any) bool +// +// compiler implements +// +// select { +// case v = <-c: +// ... foo +// default: +// ... bar +// } +// +// as +// +// if c != nil && selectnbrecv(&v, c) { +// ... foo +// } else { +// ... bar +// } +// +#pragma textflag 7 +void +runtime·selectnbrecv(byte *v, Hchan *c, bool ok) +{ + runtime·chanrecv(c, v, &ok, nil); +} + // newselect(size uint32) (sel *byte); #pragma textflag 7 void @@ -475,10 +510,11 @@ runtime·newselect(int32 size, ...) if(size > 1) n = size-1; - sel = runtime·mal(sizeof(*sel) + n*sizeof(sel->scase[0])); + sel = runtime·mal(sizeof(*sel) + n*sizeof(sel->scase[0]) + size*sizeof(sel->order[0])); sel->tcase = size; sel->ncase = 0; + sel->order = (void*)(sel->scase + size); *selp = sel; if(debug) runtime·printf("newselect s=%p size=%d\n", sel, size); @@ -619,6 +655,13 @@ selunlock(Select *sel) } } +void +runtime·block(void) +{ + g->status = Gwaiting; // forever + runtime·gosched(); +} + // selectgo(sel *byte); // // overwrites return pc on stack to signal which case of the select @@ -629,7 +672,7 @@ selunlock(Select *sel) void runtime·selectgo(Select *sel) { - uint32 p, o, i, j; + uint32 o, i, j; Scase *cas, *dfl; Hchan *c; SudoG *sg; @@ -642,29 +685,24 @@ runtime·selectgo(Select *sel) if(debug) runtime·printf("select: sel=%p\n", sel); - if(sel->ncase < 2) { - if(sel->ncase < 1) { - g->status = Gwaiting; // forever - runtime·gosched(); - } - // TODO: make special case of one. - } + // The compiler rewrites selects that statically have + // only 0 or 1 cases plus default into simpler constructs. + // The only way we can end up with such small sel->ncase + // values here is for a larger select in which most channels + // have been nilled out. The general code handles those + // cases correctly, and they are rare enough not to bother + // optimizing (and needing to test). - // select a (relative) prime - for(i=0;; i++) { - p = fastrand1(); - if(gcd(p, sel->ncase) == 1) - break; - if(i > 1000) - runtime·throw("select: failed to select prime"); + // generate permuted order + for(i=0; i<sel->ncase; i++) + sel->order[i] = i; + for(i=1; i<sel->ncase; i++) { + o = sel->order[i]; + j = fastrandn(i+1); + sel->order[i] = sel->order[j]; + sel->order[j] = o; } - // select an initial offset - o = fastrand2(); - - p %= sel->ncase; - o %= sel->ncase; - // sort the cases by Hchan address to get the locking order. for(i=1; i<sel->ncase; i++) { cas = sel->scase[i]; @@ -672,13 +710,13 @@ runtime·selectgo(Select *sel) sel->scase[j] = sel->scase[j-1]; sel->scase[j] = cas; } - sellock(sel); loop: // pass 1 - look for something already waiting dfl = nil; for(i=0; i<sel->ncase; i++) { + o = sel->order[i]; cas = sel->scase[o]; c = cas->chan; @@ -713,10 +751,6 @@ loop: dfl = cas; break; } - - o += p; - if(o >= sel->ncase) - o -= sel->ncase; } if(dfl != nil) { @@ -727,6 +761,7 @@ loop: // pass 2 - enqueue on all chans for(i=0; i<sel->ncase; i++) { + o = sel->order[i]; cas = sel->scase[o]; c = cas->chan; sg = allocsg(c); @@ -734,32 +769,15 @@ loop: switch(cas->send) { case 0: // recv - if(c->dataqsiz > 0) { - if(c->qcount > 0) - runtime·throw("select: pass 2 async recv"); - } else { - if(dequeue(&c->sendq, c)) - runtime·throw("select: pass 2 sync recv"); - } enqueue(&c->recvq, sg); break; case 1: // send - if(c->dataqsiz > 0) { - if(c->qcount < c->dataqsiz) - runtime·throw("select: pass 2 async send"); - } else { - if(dequeue(&c->recvq, c)) - runtime·throw("select: pass 2 sync send"); + if(c->dataqsiz == 0) c->elemalg->copy(c->elemsize, sg->elem, cas->u.elem); - } enqueue(&c->sendq, sg); break; } - - o += p; - if(o >= sel->ncase) - o -= sel->ncase; } g->param = nil; @@ -773,18 +791,14 @@ loop: // pass 3 - dequeue from unsuccessful chans // otherwise they stack up on quiet channels for(i=0; i<sel->ncase; i++) { - if(sg == nil || o != sg->offset) { - cas = sel->scase[o]; + if(sg == nil || i != sg->offset) { + cas = sel->scase[i]; c = cas->chan; if(cas->send) dequeueg(&c->sendq, c); else dequeueg(&c->recvq, c); } - - o += p; - if(o >= sel->ncase) - o -= sel->ncase; } if(sg == nil) @@ -858,7 +872,6 @@ rclose: if(cas->u.elemp != nil) c->elemalg->copy(c->elemsize, cas->u.elemp, nil); c->closed |= Rclosed; - incerr(c); goto retc; syncsend: @@ -871,12 +884,6 @@ syncsend: gp = sg->g; gp->param = sg; runtime·ready(gp); - goto retc; - -sclose: - // send on closed channel - incerr(c); - goto retc; retc: selunlock(sel); @@ -886,6 +893,12 @@ retc: as = (byte*)&sel + cas->so; freesel(sel); *as = true; + return; + +sclose: + // send on closed channel + selunlock(sel); + runtime·panicstring("send on closed channel"); } // closechan(sel *byte); @@ -899,7 +912,11 @@ runtime·closechan(Hchan *c) runtime·gosched(); runtime·lock(c); - incerr(c); + if(c->closed & Wclosed) { + runtime·unlock(c); + runtime·panicstring("close of closed channel"); + } + c->closed |= Wclosed; // release all readers @@ -1039,22 +1056,6 @@ freesg(Hchan *c, SudoG *sg) } static uint32 -gcd(uint32 u, uint32 v) -{ - for(;;) { - if(u > v) { - if(v == 0) - return u; - u = u%v; - continue; - } - if(u == 0) - return v; - v = v%u; - } -} - -static uint32 fastrand1(void) { static uint32 x = 0x49f6428aUL; @@ -1066,12 +1067,19 @@ fastrand1(void) } static uint32 -fastrand2(void) +fastrandn(uint32 n) { - static uint32 x = 0x49f6428aUL; + uint32 max, r; - x += x; - if(x & 0x80000000L) - x ^= 0xfafd871bUL; - return x; + if(n <= 1) + return 0; + + r = fastrand1(); + if(r < (1ULL<<31)-n) // avoid computing max in common case + return r%n; + + max = (1ULL<<31)/n * n; + while(r >= max) + r = fastrand1(); + return r%n; } diff --git a/src/pkg/runtime/chan_defs.go b/src/pkg/runtime/chan_defs.go deleted file mode 100644 index 5cfea6e15..000000000 --- a/src/pkg/runtime/chan_defs.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is chan.c - -package runtime - -type sudoG struct { - g *g_ - selgen uint32 - offset int16 - isfree int8 - link *sudoG - elem [8]byte -} - -type waitQ struct { - first *sudoG - last *sudoG -} - -type hChan struct { - qcount uint32 - dataqsiz uint32 - elemsize uint16 - closed uint16 - elemalign uint8 - elemalg *alg - senddataq *link - recvdataq *link - recvq waitQ - sendq waitQ - free sudoG - lock -} - -type link struct { - link *link - elem [8]byte -} - -type scase struct { - chan_ *hChan - pc *byte - send uint16 - so uint16 - elemp *byte // union elem [8]byte -} - -type select_ struct { - tcase uint16 - ncase uint16 - link *select_ - scase [1]*scase -} diff --git a/src/pkg/runtime/darwin/386/signal.c b/src/pkg/runtime/darwin/386/signal.c index 53a4e2f17..33f47d44f 100644 --- a/src/pkg/runtime/darwin/386/signal.c +++ b/src/pkg/runtime/darwin/386/signal.c @@ -66,6 +66,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = (uintptr)info->si_addr; + gp->sigpc = r->eip; // Only push runtime·sigpanic if r->eip != 0. // If r->eip == 0, probably panicked because of a diff --git a/src/pkg/runtime/darwin/386/sys.s b/src/pkg/runtime/darwin/386/sys.s index 79bbfb68b..7961e369c 100644 --- a/src/pkg/runtime/darwin/386/sys.s +++ b/src/pkg/runtime/darwin/386/sys.s @@ -138,7 +138,7 @@ TEXT runtime·bsdthread_create(SB),7,$32 MOVL $0x1000000, 20(SP) // flags = PTHREAD_START_CUSTOM INT $0x80 JAE 3(PC) - MOVL $-1, AX + NEGL AX RET MOVL $0, AX RET diff --git a/src/pkg/runtime/darwin/amd64/signal.c b/src/pkg/runtime/darwin/amd64/signal.c index 474a1bd5c..948b6c9c2 100644 --- a/src/pkg/runtime/darwin/amd64/signal.c +++ b/src/pkg/runtime/darwin/amd64/signal.c @@ -76,6 +76,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = (uintptr)info->si_addr; + gp->sigpc = r->rip; // Only push runtime·sigpanic if r->rip != 0. // If r->rip == 0, probably panicked because of a diff --git a/src/pkg/runtime/darwin/amd64/sys.s b/src/pkg/runtime/darwin/amd64/sys.s index 05dbc7b93..bc970156a 100644 --- a/src/pkg/runtime/darwin/amd64/sys.s +++ b/src/pkg/runtime/darwin/amd64/sys.s @@ -141,7 +141,7 @@ TEXT runtime·bsdthread_create(SB),7,$0 MOVQ $(0x2000000+360), AX // bsdthread_create SYSCALL JCC 3(PC) - MOVL $-1, AX + NEGL AX RET MOVL $0, AX RET diff --git a/src/pkg/runtime/darwin/mem.c b/src/pkg/runtime/darwin/mem.c index 7fb2c2807..cbae18718 100644 --- a/src/pkg/runtime/darwin/mem.c +++ b/src/pkg/runtime/darwin/mem.c @@ -10,10 +10,8 @@ runtime·SysAlloc(uintptr n) mstats.sys += n; v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) { - runtime·printf("mmap: errno=%p\n", v); - runtime·throw("mmap"); - } + if(v < (void*)4096) + return nil; return v; } @@ -32,8 +30,19 @@ runtime·SysFree(void *v, uintptr n) runtime·munmap(v, n); } +void* +runtime·SysReserve(void *v, uintptr n) +{ + return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); +} void -runtime·SysMemInit(void) +runtime·SysMap(void *v, uintptr n) { + void *p; + + mstats.sys += n; + p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); + if(p != v) + runtime·throw("runtime: cannot map pages in arena address space"); } diff --git a/src/pkg/runtime/darwin/runtime_defs.go b/src/pkg/runtime/darwin/runtime_defs.go deleted file mode 100644 index cf0b414a9..000000000 --- a/src/pkg/runtime/darwin/runtime_defs.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is runtime.h - -package runtime - -type lock struct { - key uint32 - sema uint32 -} - -type usema struct { - u uint32 - k uint32 -} - - -type note struct { - wakeup int32 - sema usema -} diff --git a/src/pkg/runtime/darwin/thread.c b/src/pkg/runtime/darwin/thread.c index d69c62412..57e813109 100644 --- a/src/pkg/runtime/darwin/thread.c +++ b/src/pkg/runtime/darwin/thread.c @@ -157,13 +157,17 @@ runtime·goenvs(void) void runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void)) { + int32 errno; + m->tls[0] = m->id; // so 386 asm can find it if(0){ runtime·printf("newosproc stk=%p m=%p g=%p fn=%p id=%d/%d ostk=%p\n", stk, m, g, fn, m->id, m->tls[0], &m); } - if(runtime·bsdthread_create(stk, m, g, fn) < 0) - runtime·throw("cannot create new OS thread"); + if((errno = runtime·bsdthread_create(stk, m, g, fn)) < 0) { + runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), -errno); + runtime·throw("runtime.newosproc"); + } } // Called to initialize a new m (including the bootstrap m). diff --git a/src/pkg/runtime/debug.go b/src/pkg/runtime/debug.go index 3ce35cc5b..d09db1be6 100644 --- a/src/pkg/runtime/debug.go +++ b/src/pkg/runtime/debug.go @@ -39,6 +39,7 @@ type MemStatsType struct { Sys uint64 // bytes obtained from system (should be sum of XxxSys below) Lookups uint64 // number of pointer lookups Mallocs uint64 // number of mallocs + Frees uint64 // number of frees // Main allocation heap statistics. HeapAlloc uint64 // bytes allocated and still in use @@ -56,15 +57,15 @@ type MemStatsType struct { MSpanSys uint64 MCacheInuse uint64 // mcache structures MCacheSys uint64 - MHeapMapSys uint64 // heap map BuckHashSys uint64 // profiling bucket hash table // Garbage collector statistics. - NextGC uint64 - PauseNs uint64 - NumGC uint32 - EnableGC bool - DebugGC bool + NextGC uint64 + PauseTotalNs uint64 + PauseNs [256]uint64 // most recent GC pause times + NumGC uint32 + EnableGC bool + DebugGC bool // Per-size allocation statistics. // Not locked during update; approximate. diff --git a/src/pkg/runtime/debug/Makefile b/src/pkg/runtime/debug/Makefile new file mode 100644 index 000000000..885f66aca --- /dev/null +++ b/src/pkg/runtime/debug/Makefile @@ -0,0 +1,11 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../Make.inc + +TARG=runtime/debug +GOFILES=\ + stack.go\ + +include ../../../Make.pkg diff --git a/src/pkg/runtime/debug/stack.go b/src/pkg/runtime/debug/stack.go new file mode 100644 index 000000000..e7d56ac23 --- /dev/null +++ b/src/pkg/runtime/debug/stack.go @@ -0,0 +1,90 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The debug package contains facilities for programs to debug themselves +// while they are running. +package debug + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "runtime" +) + +var ( + dunno = []byte("???") + centerDot = []byte("·") + dot = []byte(".") +) + +// PrintStack prints to standard error the stack trace returned by Stack. +func PrintStack() { + os.Stderr.Write(stack()) +} + +// Stack returns a formatted stack trace of the goroutine that calls it. +// For each routine, it includes the source line information and PC value, +// then attempts to discover, for Go functions, the calling function or +// method and the text of the line containing the invocation. +func Stack() []byte { + return stack() +} + +// stack implements Stack, skipping 2 frames +func stack() []byte { + buf := new(bytes.Buffer) // the returned data + // As we loop, we open files and read them. These variables record the currently + // loaded file. + var lines [][]byte + var lastFile string + for i := 2; ; i++ { // Caller we care about is the user, 2 frames up + pc, file, line, ok := runtime.Caller(i) + if !ok { + break + } + // Print this much at least. If we can't find the source, it won't show. + fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc) + if file != lastFile { + data, err := ioutil.ReadFile(file) + if err != nil { + continue + } + lines = bytes.Split(data, []byte{'\n'}, -1) + lastFile = file + } + line-- // in stack trace, lines are 1-indexed but our array is 0-indexed + fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line)) + } + return buf.Bytes() +} + +// source returns a space-trimmed slice of the n'th line. +func source(lines [][]byte, n int) []byte { + if n < 0 || n >= len(lines) { + return dunno + } + return bytes.Trim(lines[n], " \t") +} + +// function returns, if possible, the name of the function containing the PC. +func function(pc uintptr) []byte { + fn := runtime.FuncForPC(pc) + if fn == nil { + return dunno + } + name := []byte(fn.Name()) + // The name includes the path name to the package, which is unnecessary + // since the file name is already included. Plus, it has center dots. + // That is, we see + // runtime/debug.*T·ptrmethod + // and want + // *T.ptrmethod + if period := bytes.Index(name, dot); period >= 0 { + name = name[period+1:] + } + name = bytes.Replace(name, centerDot, dot, -1) + return name +} diff --git a/src/pkg/runtime/debug/stack_test.go b/src/pkg/runtime/debug/stack_test.go new file mode 100644 index 000000000..f4bdc4624 --- /dev/null +++ b/src/pkg/runtime/debug/stack_test.go @@ -0,0 +1,55 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package debug + +import ( + "strings" + "testing" +) + +type T int + +func (t *T) ptrmethod() []byte { + return Stack() +} +func (t T) method() []byte { + return t.ptrmethod() +} + +/* + The traceback should look something like this, modulo line numbers and hex constants. + Don't worry much about the base levels, but check the ones in our own package. + + /Users/r/go/src/pkg/runtime/debug/stack_test.go:15 (0x13878) + *T.ptrmethod: return Stack() + /Users/r/go/src/pkg/runtime/debug/stack_test.go:18 (0x138dd) + T.method: return t.ptrmethod() + /Users/r/go/src/pkg/runtime/debug/stack_test.go:23 (0x13920) + TestStack: b := T(0).method() + /Users/r/go/src/pkg/testing/testing.go:132 (0x14a7a) + tRunner: test.F(t) + /Users/r/go/src/pkg/runtime/proc.c:145 (0xc970) + ???: runtime·unlock(&runtime·sched); +*/ +func TestStack(t *testing.T) { + b := T(0).method() + lines := strings.Split(string(b), "\n", -1) + if len(lines) <= 6 { + t.Fatal("too few lines") + } + check(t, lines[0], "src/pkg/runtime/debug/stack_test.go") + check(t, lines[1], "\t*T.ptrmethod: return Stack()") + check(t, lines[2], "src/pkg/runtime/debug/stack_test.go") + check(t, lines[3], "\tT.method: return t.ptrmethod()") + check(t, lines[4], "src/pkg/runtime/debug/stack_test.go") + check(t, lines[5], "\tTestStack: b := T(0).method()") + check(t, lines[6], "src/pkg/testing/testing.go") +} + +func check(t *testing.T, line, has string) { + if strings.Index(line, has) < 0 { + t.Errorf("expected %q in %q", has, line) + } +} diff --git a/src/pkg/runtime/extern.go b/src/pkg/runtime/extern.go index 77c3e8e3a..dba28324c 100644 --- a/src/pkg/runtime/extern.go +++ b/src/pkg/runtime/extern.go @@ -31,6 +31,19 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) // It returns the number of entries written to pc. func Callers(skip int, pc []uintptr) int +type Func struct { // Keep in sync with runtime.h:struct Func + name string + typ string // go type string + src string // src file name + pcln []byte // pc/ln tab for this func + entry uintptr // entry pc + pc0 uintptr // starting pc, ln for table + ln0 int32 + frame int32 // stack frame size + args int32 // number of 32-bit in/out args + locals int32 // number of 32-bit locals +} + // FuncForPC returns a *Func describing the function that contains the // given program counter address, or else nil. func FuncForPC(pc uintptr) *Func diff --git a/src/pkg/runtime/freebsd/386/signal.c b/src/pkg/runtime/freebsd/386/signal.c index 52b820df1..ddb11fc3b 100644 --- a/src/pkg/runtime/freebsd/386/signal.c +++ b/src/pkg/runtime/freebsd/386/signal.c @@ -63,6 +63,7 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = (uintptr)info->si_addr; + gp->sigpc = r->mc_eip; // Only push runtime·sigpanic if r->mc_eip != 0. // If r->mc_eip == 0, probably panicked because of a diff --git a/src/pkg/runtime/freebsd/amd64/signal.c b/src/pkg/runtime/freebsd/amd64/signal.c index c74ddad0b..9f873d276 100644 --- a/src/pkg/runtime/freebsd/amd64/signal.c +++ b/src/pkg/runtime/freebsd/amd64/signal.c @@ -71,6 +71,7 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = (uintptr)info->si_addr; + gp->sigpc = r->mc_rip; // Only push runtime·sigpanic if r->mc_rip != 0. // If r->mc_rip == 0, probably panicked because of a diff --git a/src/pkg/runtime/freebsd/mem.c b/src/pkg/runtime/freebsd/mem.c index 7fb2c2807..cbae18718 100644 --- a/src/pkg/runtime/freebsd/mem.c +++ b/src/pkg/runtime/freebsd/mem.c @@ -10,10 +10,8 @@ runtime·SysAlloc(uintptr n) mstats.sys += n; v = runtime·mmap(nil, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, -1, 0); - if(v < (void*)4096) { - runtime·printf("mmap: errno=%p\n", v); - runtime·throw("mmap"); - } + if(v < (void*)4096) + return nil; return v; } @@ -32,8 +30,19 @@ runtime·SysFree(void *v, uintptr n) runtime·munmap(v, n); } +void* +runtime·SysReserve(void *v, uintptr n) +{ + return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); +} void -runtime·SysMemInit(void) +runtime·SysMap(void *v, uintptr n) { + void *p; + + mstats.sys += n; + p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); + if(p != v) + runtime·throw("runtime: cannot map pages in arena address space"); } diff --git a/src/pkg/runtime/freebsd/runtime_defs.go b/src/pkg/runtime/freebsd/runtime_defs.go deleted file mode 100644 index 86de13316..000000000 --- a/src/pkg/runtime/freebsd/runtime_defs.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// OS-Specific Go definitions of internal structures. Master is runtime.h - -package runtime - -type lock struct { - key uint32 - sema uint32 -} - -type note lock diff --git a/src/pkg/runtime/hashmap.c b/src/pkg/runtime/hashmap.c index a03202ed6..f0d5ce90a 100644 --- a/src/pkg/runtime/hashmap.c +++ b/src/pkg/runtime/hashmap.c @@ -9,7 +9,7 @@ /* Return a pointer to the struct/union of type "type" whose "field" field is addressed by pointer "p". */ -struct hash { /* a hash table; initialize with hash_init() */ +struct Hmap { /* a hash table; initialize with hash_init() */ uint32 count; /* elements in table - must be first */ uint8 datasize; /* amount of data to store in entry */ @@ -82,7 +82,7 @@ struct hash_subtable { /* return a hash layer with 2**power empty entries */ static struct hash_subtable * -hash_subtable_new (struct hash *h, int32 power, int32 used) +hash_subtable_new (Hmap *h, int32 power, int32 used) { int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); int32 bytes = elemsize << power; @@ -127,7 +127,7 @@ init_sizes (int64 hint, int32 *init_power, int32 *max_power) } static void -hash_init (struct hash *h, +hash_init (Hmap *h, int32 datasize, hash_hash_t (*data_hash) (uint32, void *), uint32 (*data_eq) (uint32, void *, void *), @@ -200,10 +200,10 @@ hash_remove_n (struct hash_subtable *st, struct hash_entry *dst_e, int32 n) static int32 hash_insert_internal (struct hash_subtable **pst, int32 flags, hash_hash_t hash, - struct hash *h, void *data, void **pres); + Hmap *h, void *data, void **pres); static void -hash_conv (struct hash *h, +hash_conv (Hmap *h, struct hash_subtable *st, int32 flags, hash_hash_t hash, struct hash_entry *e) @@ -266,7 +266,7 @@ hash_conv (struct hash *h, } static void -hash_grow (struct hash *h, struct hash_subtable **pst, int32 flags) +hash_grow (Hmap *h, struct hash_subtable **pst, int32 flags) { struct hash_subtable *old_st = *pst; int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); @@ -290,7 +290,7 @@ hash_grow (struct hash *h, struct hash_subtable **pst, int32 flags) } static int32 -hash_lookup (struct hash *h, void *data, void **pres) +hash_lookup (Hmap *h, void *data, void **pres) { int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); hash_hash_t hash = (*h->data_hash) (h->keysize, data) & ~HASH_MASK; @@ -331,7 +331,7 @@ hash_lookup (struct hash *h, void *data, void **pres) } static int32 -hash_remove (struct hash *h, void *data, void *arg) +hash_remove (Hmap *h, void *data, void *arg) { int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); hash_hash_t hash = (*h->data_hash) (h->keysize, data) & ~HASH_MASK; @@ -374,7 +374,7 @@ hash_remove (struct hash *h, void *data, void *arg) static int32 hash_insert_internal (struct hash_subtable **pst, int32 flags, hash_hash_t hash, - struct hash *h, void *data, void **pres) + Hmap *h, void *data, void **pres) { int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); @@ -455,7 +455,7 @@ hash_insert_internal (struct hash_subtable **pst, int32 flags, hash_hash_t hash, } static int32 -hash_insert (struct hash *h, void *data, void **pres) +hash_insert (Hmap *h, void *data, void **pres) { int32 rc = hash_insert_internal (&h->st, 0, (*h->data_hash) (h->keysize, data), h, data, pres); @@ -464,7 +464,7 @@ hash_insert (struct hash *h, void *data, void **pres) } static uint32 -hash_count (struct hash *h) +hash_count (Hmap *h) { return (h->count); } @@ -571,7 +571,7 @@ hash_next (struct hash_iter *it) } static void -hash_iter_init (struct hash *h, struct hash_iter *it) +hash_iter_init (Hmap *h, struct hash_iter *it) { it->elemsize = h->datasize + offsetof (struct hash_entry, data[0]); it->changes = h->changes; @@ -607,7 +607,7 @@ clean_st (struct hash_subtable *st, int32 *slots, int32 *used) } static void -hash_destroy (struct hash *h) +hash_destroy (Hmap *h) { int32 slots = 0; int32 used = 0; @@ -646,7 +646,7 @@ hash_visit_internal (struct hash_subtable *st, } static void -hash_visit (struct hash *h, void (*data_visit) (void *arg, int32 level, void *data), void *arg) +hash_visit (Hmap *h, void (*data_visit) (void *arg, int32 level, void *data), void *arg) { hash_visit_internal (h->st, 0, 0, data_visit, arg); } diff --git a/src/pkg/runtime/hashmap.h b/src/pkg/runtime/hashmap.h index 0737535b5..d0fd3527f 100644 --- a/src/pkg/runtime/hashmap.h +++ b/src/pkg/runtime/hashmap.h @@ -70,7 +70,7 @@ #define free(x) runtime·free(x) #define memmove(a,b,c) runtime·memmove(a, b, c) -struct hash; /* opaque */ +struct Hmap; /* opaque */ struct hash_subtable; /* opaque */ struct hash_entry; /* opaque */ @@ -83,7 +83,7 @@ struct hash_iter { int32 changes; /* number of changes observed last time */ int32 i; /* stack pointer in subtable_state */ hash_hash_t last_hash; /* last hash value returned */ - struct hash *h; /* the hash table */ + struct Hmap *h; /* the hash table */ struct hash_iter_sub { struct hash_entry *e; /* pointer into subtable */ struct hash_entry *start; /* start of subtable */ diff --git a/src/pkg/runtime/hashmap_defs.go b/src/pkg/runtime/hashmap_defs.go deleted file mode 100644 index 57780df87..000000000 --- a/src/pkg/runtime/hashmap_defs.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is hashmap.[c,h] - -package runtime - -type hash_hash uintptr - -type hash_entry struct { - hash hash_hash - key byte // dwarf.c substitutes the real type - val byte // for key and val -} - -type hash_subtable struct { - power uint8 - used uint8 - datasize uint8 - max_probes uint8 - limit_bytes int16 - end *hash_entry - entry hash_entry // TODO: [0]hash_entry -} - -type hash struct { - count uint32 - datasize uint8 - max_power uint8 - max_probes uint8 - indirectval uint8 - changes int32 - data_hash func(uint32, uintptr) hash_hash - data_eq func(uint32, uintptr, uintptr) uint32 - data_del func(uint32, uintptr, uintptr) - st *hash_subtable - keysize uint32 - valsize uint32 - datavo uint32 - ko0 uint32 - vo0 uint32 - ko1 uint32 - vo1 uint32 - po1 uint32 - ko2 uint32 - vo2 uint32 - po2 uint32 - keyalg *alg - valalg *alg -} diff --git a/src/pkg/runtime/iface_defs.go b/src/pkg/runtime/iface_defs.go deleted file mode 100644 index 69d52ef9a..000000000 --- a/src/pkg/runtime/iface_defs.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -/* - * Must match iface.c:/Itable and compilers. - * NOTE: type.go has an Itable, that is the version of Itab used by the reflection code. - */ -type itab struct { - Itype *Type - Type *Type - link *itab - bad int32 - unused int32 - Fn func() // TODO: [0]func() -} diff --git a/src/pkg/runtime/linux/386/signal.c b/src/pkg/runtime/linux/386/signal.c index 0dbfcf9ff..9651a6f28 100644 --- a/src/pkg/runtime/linux/386/signal.c +++ b/src/pkg/runtime/linux/386/signal.c @@ -60,6 +60,7 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = ((uintptr*)info)[3]; + gp->sigpc = r->eip; // Only push runtime·sigpanic if r->eip != 0. // If r->eip == 0, probably panicked because of a diff --git a/src/pkg/runtime/linux/amd64/signal.c b/src/pkg/runtime/linux/amd64/signal.c index e78bbda9d..9e501c96d 100644 --- a/src/pkg/runtime/linux/amd64/signal.c +++ b/src/pkg/runtime/linux/amd64/signal.c @@ -70,6 +70,7 @@ runtime·sighandler(int32 sig, Siginfo* info, void* context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = ((uintptr*)info)[2]; + gp->sigpc = r->rip; // Only push runtime·sigpanic if r->rip != 0. // If r->rip == 0, probably panicked because of a diff --git a/src/pkg/runtime/linux/arm/signal.c b/src/pkg/runtime/linux/arm/signal.c index c65aff913..481bd13c6 100644 --- a/src/pkg/runtime/linux/arm/signal.c +++ b/src/pkg/runtime/linux/arm/signal.c @@ -67,6 +67,7 @@ runtime·sighandler(int32 sig, Siginfo *info, void *context) gp->sig = sig; gp->sigcode0 = info->si_code; gp->sigcode1 = r->fault_address; + gp->sigpc = r->arm_pc; // If this is a leaf function, we do smash LR, // but we're not going back there anyway. diff --git a/src/pkg/runtime/linux/mem.c b/src/pkg/runtime/linux/mem.c index e750f97ea..3a83e7394 100644 --- a/src/pkg/runtime/linux/mem.c +++ b/src/pkg/runtime/linux/mem.c @@ -12,12 +12,11 @@ runtime·SysAlloc(uintptr n) p = runtime·mmap(nil, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, -1, 0); if(p < (void*)4096) { if(p == (void*)EACCES) { - runtime·printf("mmap: access denied\n"); - runtime·printf("If you're running SELinux, enable execmem for this process.\n"); + runtime·printf("runtime: mmap: access denied\n"); + runtime·printf("if you're running SELinux, enable execmem for this process.\n"); runtime·exit(2); } - runtime·printf("mmap: errno=%p\n", p); - runtime·throw("mmap"); + return nil; } return p; } @@ -37,7 +36,19 @@ runtime·SysFree(void *v, uintptr n) runtime·munmap(v, n); } +void* +runtime·SysReserve(void *v, uintptr n) +{ + return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); +} + void -runtime·SysMemInit(void) +runtime·SysMap(void *v, uintptr n) { + void *p; + + mstats.sys += n; + p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); + if(p != v) + runtime·throw("runtime: cannot map pages in arena address space"); } diff --git a/src/pkg/runtime/linux/runtime_defs.go b/src/pkg/runtime/linux/runtime_defs.go deleted file mode 100644 index 86de13316..000000000 --- a/src/pkg/runtime/linux/runtime_defs.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// OS-Specific Go definitions of internal structures. Master is runtime.h - -package runtime - -type lock struct { - key uint32 - sema uint32 -} - -type note lock diff --git a/src/pkg/runtime/linux/thread.c b/src/pkg/runtime/linux/thread.c index 979260ba1..d5f9a8fb0 100644 --- a/src/pkg/runtime/linux/thread.c +++ b/src/pkg/runtime/linux/thread.c @@ -238,8 +238,7 @@ runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void)) /* * note: strace gets confused if we use CLONE_PTRACE here. */ - flags = CLONE_PARENT /* getppid doesn't change in child */ - | CLONE_VM /* share memory */ + flags = CLONE_VM /* share memory */ | CLONE_FS /* share cwd, etc */ | CLONE_FILES /* share fd table */ | CLONE_SIGHAND /* share sig handler table */ diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc index f5ca9f918..cc28b943d 100644 --- a/src/pkg/runtime/malloc.goc +++ b/src/pkg/runtime/malloc.goc @@ -175,7 +175,7 @@ runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp, uint32 **ref) MSpan *s; mstats.nlookup++; - s = runtime·MHeap_LookupMaybe(&runtime·mheap, (uintptr)v>>PageShift); + s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); if(sp) *sp = s; if(s == nil) { @@ -249,8 +249,45 @@ int32 runtime·sizeof_C_MStats = sizeof(MStats); void runtime·mallocinit(void) { - runtime·SysMemInit(); + byte *p; + uintptr arena_size; + runtime·InitSizes(); + + if(sizeof(void*) == 8) { + // On a 64-bit machine, allocate from a single contiguous reservation. + // 16 GB should be big enough for now. + // + // The code will work with the reservation at any address, but ask + // SysReserve to use 0x000000f800000000 if possible. + // Allocating a 16 GB region takes away 36 bits, and the amd64 + // doesn't let us choose the top 17 bits, so that leaves the 11 bits + // in the middle of 0x00f8 for us to choose. Choosing 0x00f8 means + // that the valid memory addresses will begin 0x00f8, 0x00f9, 0x00fa, 0x00fb. + // None of the bytes f8 f9 fa fb can appear in valid UTF-8, and + // they are otherwise as far from ff (likely a common byte) as possible. + // Choosing 0x00 for the leading 6 bits was more arbitrary, but it + // is not a common ASCII code point either. Using 0x11f8 instead + // caused out of memory errors on OS X during thread allocations. + // These choices are both for debuggability and to reduce the + // odds of the conservative garbage collector not collecting memory + // because some non-pointer block of memory had a bit pattern + // that matched a memory address. + arena_size = 16LL<<30; + p = runtime·SysReserve((void*)(0x00f8ULL<<32), arena_size); + if(p == nil) + runtime·throw("runtime: cannot reserve arena virtual address space"); + runtime·mheap.arena_start = p; + runtime·mheap.arena_used = p; + runtime·mheap.arena_end = p + arena_size; + } else { + // On a 32-bit machine, we'll take what we can get for each allocation + // and maintain arena_start and arena_end as min, max we've seen. + runtime·mheap.arena_start = (byte*)0xffffffff; + runtime·mheap.arena_end = 0; + } + + // Initialize the rest of the allocator. runtime·MHeap_Init(&runtime·mheap, runtime·SysAlloc); m->mcache = runtime·allocmcache(); @@ -258,6 +295,32 @@ runtime·mallocinit(void) runtime·free(runtime·malloc(1)); } +void* +runtime·MHeap_SysAlloc(MHeap *h, uintptr n) +{ + byte *p; + + if(sizeof(void*) == 8) { + // Keep taking from our reservation. + if(h->arena_end - h->arena_used < n) + return nil; + p = h->arena_used; + runtime·SysMap(p, n); + h->arena_used += n; + return p; + } else { + // Take what we can get from the OS. + p = runtime·SysAlloc(n); + if(p == nil) + return nil; + if(p+n > h->arena_used) + h->arena_used = p+n; + if(p > h->arena_end) + h->arena_end = p; + return p; + } +} + // Runtime stubs. void* @@ -282,13 +345,17 @@ static struct { FixAlloc; } stacks; +enum { + FixedStack = StackBig + StackExtra +}; + void* runtime·stackalloc(uint32 n) { void *v; uint32 *ref; - if(m->mallocing || m->gcing) { + if(m->mallocing || m->gcing || n == FixedStack) { runtime·lock(&stacks); if(stacks.size == 0) runtime·FixAlloc_Init(&stacks, n, runtime·SysAlloc, nil, nil); @@ -310,9 +377,9 @@ runtime·stackalloc(uint32 n) } void -runtime·stackfree(void *v) +runtime·stackfree(void *v, uintptr n) { - if(m->mallocing || m->gcing) { + if(m->mallocing || m->gcing || n == FixedStack) { runtime·lock(&stacks); runtime·FixAlloc_Free(&stacks, v); mstats.stacks_inuse = stacks.inuse; diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h index 0cee6c0dd..e2472e8d2 100644 --- a/src/pkg/runtime/malloc.h +++ b/src/pkg/runtime/malloc.h @@ -19,7 +19,6 @@ // used to manage storage used by the allocator. // MHeap: the malloc heap, managed at page (4096-byte) granularity. // MSpan: a run of pages managed by the MHeap. -// MHeapMap: a mapping from page IDs to MSpans. // MCentral: a shared free list for a given size class. // MCache: a per-thread (in Go, per-M) cache for small objects. // MStats: allocation statistics. @@ -84,7 +83,6 @@ typedef struct FixAlloc FixAlloc; typedef struct MCentral MCentral; typedef struct MHeap MHeap; -typedef struct MHeapMap MHeapMap; typedef struct MSpan MSpan; typedef struct MStats MStats; typedef struct MLink MLink; @@ -108,13 +106,16 @@ enum MaxMCacheSize = 2<<20, // Maximum bytes in one MCache MaxMHeapList = 1<<(20 - PageShift), // Maximum page length for fixed-size list in MHeap. HeapAllocChunk = 1<<20, // Chunk size for heap growth -}; + // Number of bits in page to span calculations (4k pages). + // On 64-bit, we limit the arena to 16G, so 22 bits suffices. + // On 32-bit, we don't bother limiting anything: 20 bits for 4G. #ifdef _64BIT -#include "mheapmap64.h" + MHeapMap_Bits = 22, #else -#include "mheapmap32.h" + MHeapMap_Bits = 20, #endif +}; // A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).) struct MLink @@ -124,7 +125,8 @@ struct MLink // SysAlloc obtains a large chunk of zeroed memory from the // operating system, typically on the order of a hundred kilobytes -// or a megabyte. +// or a megabyte. If the pointer argument is non-nil, the caller +// wants a mapping there or nowhere. // // SysUnused notifies the operating system that the contents // of the memory region are no longer needed and can be reused @@ -134,11 +136,19 @@ struct MLink // SysFree returns it unconditionally; this is only used if // an out-of-memory error has been detected midway through // an allocation. It is okay if SysFree is a no-op. +// +// SysReserve reserves address space without allocating memory. +// If the pointer passed to it is non-nil, the caller wants the +// reservation there, but SysReserve can still choose another +// location if that one is unavailable. +// +// SysMap maps previously reserved address space for use. void* runtime·SysAlloc(uintptr nbytes); void runtime·SysFree(void *v, uintptr nbytes); void runtime·SysUnused(void *v, uintptr nbytes); -void runtime·SysMemInit(void); +void runtime·SysMap(void *v, uintptr nbytes); +void* runtime·SysReserve(void *v, uintptr nbytes); // FixAlloc is a simple free-list allocator for fixed size objects. // Malloc uses a FixAlloc wrapped around SysAlloc to manages its @@ -176,6 +186,7 @@ struct MStats uint64 sys; // bytes obtained from system (should be sum of xxx_sys below) uint64 nlookup; // number of pointer lookups uint64 nmalloc; // number of mallocs + uint64 nfree; // number of frees // Statistics about malloc heap. // protected by mheap.Lock @@ -193,13 +204,13 @@ struct MStats uint64 mspan_sys; uint64 mcache_inuse; // MCache structures uint64 mcache_sys; - uint64 heapmap_sys; // heap map uint64 buckhash_sys; // profiling bucket hash table // Statistics about garbage collector. // Protected by stopping the world during GC. uint64 next_gc; // next GC (in heap_alloc time) - uint64 pause_ns; + uint64 pause_total_ns; + uint64 pause_ns[256]; uint32 numgc; bool enablegc; bool debuggc; @@ -321,11 +332,13 @@ struct MHeap MSpan *allspans; // span lookup - MHeapMap map; + MSpan *map[1<<MHeapMap_Bits]; // range of addresses we might see in the heap - byte *min; - byte *max; + byte *bitmap; + byte *arena_start; + byte *arena_used; + byte *arena_end; // central free lists for small size classes. // the union makes sure that the MCentrals are @@ -344,18 +357,15 @@ extern MHeap runtime·mheap; void runtime·MHeap_Init(MHeap *h, void *(*allocator)(uintptr)); MSpan* runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, int32 acct); void runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct); -MSpan* runtime·MHeap_Lookup(MHeap *h, PageID p); -MSpan* runtime·MHeap_LookupMaybe(MHeap *h, PageID p); +MSpan* runtime·MHeap_Lookup(MHeap *h, void *v); +MSpan* runtime·MHeap_LookupMaybe(MHeap *h, void *v); void runtime·MGetSizeClassInfo(int32 sizeclass, int32 *size, int32 *npages, int32 *nobj); +void* runtime·MHeap_SysAlloc(MHeap *h, uintptr n); void* runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed); int32 runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **s, uint32 **ref); void runtime·gc(int32 force); -void* runtime·SysAlloc(uintptr); -void runtime·SysUnused(void*, uintptr); -void runtime·SysFree(void*, uintptr); - enum { RefcountOverhead = 4, // one uint32 per object diff --git a/src/pkg/runtime/malloc_defs.go b/src/pkg/runtime/malloc_defs.go deleted file mode 100644 index bfb96f409..000000000 --- a/src/pkg/runtime/malloc_defs.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is malloc.h - -package runtime - -import "unsafe" - -const ( - pageShift = 12 - pageSize = 1 << pageShift - pageMask = pageSize - 1 -) - -type pageID uintptr - -const ( - numSizeClasses = 67 - maxSmallSize = 32 << 10 - fixAllocChunk = 128 << 10 - maxMCacheListLen = 256 - maxMCacheSize = 2 << 20 - maxMHeapList = 1 << 8 // 1 << (20 - pageShift) - heapAllocChunk = 1 << 20 -) - -type mLink struct { - next *mLink -} - -type fixAlloc struct { - size uintptr - alloc func(uintptr) - first func(unsafe.Pointer, *byte) - arg unsafe.Pointer - list *mLink - chunk *byte - nchunk uint32 - inuse uintptr - sys uintptr -} - - -// MStats? used to be in extern.go - -type mCacheList struct { - list *mLink - nlist uint32 - nlistmin uint32 -} - -type mCache struct { - list [numSizeClasses]mCacheList - size uint64 - local_alloc int64 - local_objects int64 - next_sample int32 -} - -type mSpan struct { - next *mSpan - prev *mSpan - allnext *mSpan - start pageID - npages uintptr - freelist *mLink - ref uint32 - sizeclass uint32 - state uint32 - // union { - gcref *uint32 // sizeclass > 0 - // gcref0 uint32; // sizeclass == 0 - // } -} - -type mCentral struct { - lock - sizeclass int32 - nonempty mSpan - empty mSpan - nfree int32 -} - -type mHeap struct { - lock - free [maxMHeapList]mSpan - large mSpan - allspans *mSpan - map_ mHeapMap - min *byte - max *byte - closure_min *byte - closure_max *byte - - central [numSizeClasses]struct { - pad [64]byte - // union: mCentral - } - - spanalloc fixAlloc - cachealloc fixAlloc -} - -const ( - refFree = iota - refStack - refNone - refSome - refcountOverhead = 4 - refNoPointers = 0x80000000 - refHasFinalizer = 0x40000000 - refProfiled = 0x20000000 - refNoProfiling = 0x10000000 - refFlags = 0xFFFF0000 -) - -const ( - mProf_None = iota - mProf_Sample - mProf_All -) - -type finalizer struct { - next *finalizer - fn func(unsafe.Pointer) - arg unsafe.Pointer - nret int32 -} diff --git a/src/pkg/runtime/mcentral.c b/src/pkg/runtime/mcentral.c index 8855dc663..f1ad119d3 100644 --- a/src/pkg/runtime/mcentral.c +++ b/src/pkg/runtime/mcentral.c @@ -118,8 +118,7 @@ MCentral_Free(MCentral *c, void *v) int32 size; // Find span for v. - page = (uintptr)v >> PageShift; - s = runtime·MHeap_Lookup(&runtime·mheap, page); + s = runtime·MHeap_Lookup(&runtime·mheap, v); if(s == nil || s->ref == 0) runtime·throw("invalid free"); diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index 6dcb61091..af1c721e8 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -76,7 +76,7 @@ scanblock(byte *b, int64 n) obj = vp[i]; if(obj == nil) continue; - if(runtime·mheap.min <= (byte*)obj && (byte*)obj < runtime·mheap.max) { + if(runtime·mheap.arena_start <= (byte*)obj && (byte*)obj < runtime·mheap.arena_end) { if(runtime·mlookup(obj, &obj, &size, nil, &refp)) { ref = *refp; switch(ref & ~RefFlags) { @@ -210,6 +210,7 @@ sweepspan(MSpan *s) case RefNone: // Free large object. mstats.alloc -= s->npages<<PageShift; + mstats.nfree++; runtime·memclr(p, s->npages<<PageShift); if(ref & RefProfiled) runtime·MProf_Free(p, s->npages<<PageShift); @@ -251,6 +252,7 @@ sweepspan(MSpan *s) if(size > sizeof(uintptr)) ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed" mstats.alloc -= size; + mstats.nfree++; mstats.by_size[s->sizeclass].nfree++; runtime·MCache_Free(c, p, s->sizeclass, size); break; @@ -381,7 +383,8 @@ runtime·gc(int32 force) t1 = runtime·nanotime(); mstats.numgc++; - mstats.pause_ns += t1 - t0; + mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t1 - t0; + mstats.pause_total_ns += t1 - t0; if(mstats.debuggc) runtime·printf("pause %D\n", t1-t0); runtime·semrelease(&gcsema); diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c index 4bb7f14e3..0c9ac0a09 100644 --- a/src/pkg/runtime/mheap.c +++ b/src/pkg/runtime/mheap.c @@ -41,7 +41,6 @@ runtime·MHeap_Init(MHeap *h, void *(*alloc)(uintptr)) runtime·FixAlloc_Init(&h->spanalloc, sizeof(MSpan), alloc, RecordSpan, h); runtime·FixAlloc_Init(&h->cachealloc, sizeof(MCache), alloc, nil, nil); - runtime·MHeapMap_Init(&h->map, alloc); // h->mapcache needs no init for(i=0; i<nelem(h->free); i++) runtime·MSpanList_Init(&h->free[i]); @@ -79,6 +78,7 @@ MHeap_AllocLocked(MHeap *h, uintptr npage, int32 sizeclass) { uintptr n; MSpan *s, *t; + PageID p; // Try in fixed-size lists up to max. for(n=npage; n < nelem(h->free); n++) { @@ -112,18 +112,29 @@ HaveSpan: mstats.mspan_sys = h->spanalloc.sys; runtime·MSpan_Init(t, s->start + npage, s->npages - npage); s->npages = npage; - runtime·MHeapMap_Set(&h->map, t->start - 1, s); - runtime·MHeapMap_Set(&h->map, t->start, t); - runtime·MHeapMap_Set(&h->map, t->start + t->npages - 1, t); + p = t->start; + if(sizeof(void*) == 8) + p -= ((uintptr)h->arena_start>>PageShift); + if(p > 0) + h->map[p-1] = s; + h->map[p] = t; + h->map[p+t->npages-1] = t; + *(uintptr*)(t->start<<PageShift) = *(uintptr*)(s->start<<PageShift); // copy "needs zeroing" mark t->state = MSpanInUse; MHeap_FreeLocked(h, t); } + if(*(uintptr*)(s->start<<PageShift) != 0) + runtime·memclr((byte*)(s->start<<PageShift), s->npages<<PageShift); + // Record span info, because gc needs to be // able to map interior pointer to containing span. s->sizeclass = sizeclass; + p = s->start; + if(sizeof(void*) == 8) + p -= ((uintptr)h->arena_start>>PageShift); for(n=0; n<npage; n++) - runtime·MHeapMap_Set(&h->map, s->start+n, s); + h->map[p+n] = s; return s; } @@ -161,6 +172,7 @@ MHeap_Grow(MHeap *h, uintptr npage) uintptr ask; void *v; MSpan *s; + PageID p; // Ask for a big chunk, to reduce the number of mappings // the operating system needs to track; also amortizes @@ -171,29 +183,21 @@ MHeap_Grow(MHeap *h, uintptr npage) if(ask < HeapAllocChunk) ask = HeapAllocChunk; - v = runtime·SysAlloc(ask); + v = runtime·MHeap_SysAlloc(h, ask); if(v == nil) { if(ask > (npage<<PageShift)) { ask = npage<<PageShift; - v = runtime·SysAlloc(ask); + v = runtime·MHeap_SysAlloc(h, ask); } if(v == nil) return false; } mstats.heap_sys += ask; - if((byte*)v < h->min || h->min == nil) - h->min = v; - if((byte*)v+ask > h->max) - h->max = (byte*)v+ask; - - // NOTE(rsc): In tcmalloc, if we've accumulated enough - // system allocations, the heap map gets entirely allocated - // in 32-bit mode. (In 64-bit mode that's not practical.) - if(!runtime·MHeapMap_Preallocate(&h->map, ((uintptr)v>>PageShift) - 1, (ask>>PageShift) + 2)) { - runtime·SysFree(v, ask); - return false; - } + if((byte*)v < h->arena_start || h->arena_start == nil) + h->arena_start = v; + if((byte*)v+ask > h->arena_end) + h->arena_end = (byte*)v+ask; // Create a fake "in use" span and free it, so that the // right coalescing happens. @@ -201,35 +205,50 @@ MHeap_Grow(MHeap *h, uintptr npage) mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; runtime·MSpan_Init(s, (uintptr)v>>PageShift, ask>>PageShift); - runtime·MHeapMap_Set(&h->map, s->start, s); - runtime·MHeapMap_Set(&h->map, s->start + s->npages - 1, s); + p = s->start; + if(sizeof(void*) == 8) + p -= ((uintptr)h->arena_start>>PageShift); + h->map[p] = s; + h->map[p + s->npages - 1] = s; s->state = MSpanInUse; MHeap_FreeLocked(h, s); return true; } -// Look up the span at the given page number. -// Page number is guaranteed to be in map +// Look up the span at the given address. +// Address is guaranteed to be in map // and is guaranteed to be start or end of span. MSpan* -runtime·MHeap_Lookup(MHeap *h, PageID p) +runtime·MHeap_Lookup(MHeap *h, void *v) { - return runtime·MHeapMap_Get(&h->map, p); + uintptr p; + + p = (uintptr)v; + if(sizeof(void*) == 8) + p -= (uintptr)h->arena_start; + return h->map[p >> PageShift]; } -// Look up the span at the given page number. -// Page number is *not* guaranteed to be in map +// Look up the span at the given address. +// Address is *not* guaranteed to be in map // and may be anywhere in the span. // Map entries for the middle of a span are only // valid for allocated spans. Free spans may have // other garbage in their middles, so we have to // check for that. MSpan* -runtime·MHeap_LookupMaybe(MHeap *h, PageID p) +runtime·MHeap_LookupMaybe(MHeap *h, void *v) { MSpan *s; + PageID p, q; - s = runtime·MHeapMap_GetMaybe(&h->map, p); + if((byte*)v < h->arena_start || (byte*)v >= h->arena_used) + return nil; + p = (uintptr)v>>PageShift; + q = p; + if(sizeof(void*) == 8) + q -= (uintptr)h->arena_start >> PageShift; + s = h->map[q]; if(s == nil || p < s->start || p - s->start >= s->npages) return nil; if(s->state != MSpanInUse) @@ -258,7 +277,9 @@ runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct) static void MHeap_FreeLocked(MHeap *h, MSpan *s) { + uintptr *sp, *tp; MSpan *t; + PageID p; if(s->state != MSpanInUse || s->ref != 0) { runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref); @@ -266,21 +287,30 @@ MHeap_FreeLocked(MHeap *h, MSpan *s) } s->state = MSpanFree; runtime·MSpanList_Remove(s); + sp = (uintptr*)(s->start<<PageShift); // Coalesce with earlier, later spans. - if((t = runtime·MHeapMap_Get(&h->map, s->start - 1)) != nil && t->state != MSpanInUse) { + p = s->start; + if(sizeof(void*) == 8) + p -= (uintptr)h->arena_start >> PageShift; + if(p > 0 && (t = h->map[p-1]) != nil && t->state != MSpanInUse) { + tp = (uintptr*)(t->start<<PageShift); + *tp |= *sp; // propagate "needs zeroing" mark s->start = t->start; s->npages += t->npages; - runtime·MHeapMap_Set(&h->map, s->start, s); + p -= t->npages; + h->map[p] = s; runtime·MSpanList_Remove(t); t->state = MSpanDead; runtime·FixAlloc_Free(&h->spanalloc, t); mstats.mspan_inuse = h->spanalloc.inuse; mstats.mspan_sys = h->spanalloc.sys; } - if((t = runtime·MHeapMap_Get(&h->map, s->start + s->npages)) != nil && t->state != MSpanInUse) { + if(p+s->npages < nelem(h->map) && (t = h->map[p+s->npages]) != nil && t->state != MSpanInUse) { + tp = (uintptr*)(t->start<<PageShift); + *sp |= *tp; // propagate "needs zeroing" mark s->npages += t->npages; - runtime·MHeapMap_Set(&h->map, s->start + s->npages - 1, s); + h->map[p + s->npages - 1] = s; runtime·MSpanList_Remove(t); t->state = MSpanDead; runtime·FixAlloc_Free(&h->spanalloc, t); diff --git a/src/pkg/runtime/mheapmap32.c b/src/pkg/runtime/mheapmap32.c deleted file mode 100644 index 323f8b87a..000000000 --- a/src/pkg/runtime/mheapmap32.c +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Heap map, 32-bit version -// See malloc.h and mheap.c for overview. - -#include "runtime.h" -#include "malloc.h" - -// 3-level radix tree mapping page ids to Span*. -void -runtime·MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)) -{ - m->allocator = allocator; -} - -MSpan* -runtime·MHeapMap_Get(MHeapMap *m, PageID k) -{ - int32 i1, i2; - - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime·throw("MHeapMap_Get"); - - return m->p[i1]->s[i2]; -} - -MSpan* -runtime·MHeapMap_GetMaybe(MHeapMap *m, PageID k) -{ - int32 i1, i2; - MHeapMapNode2 *p2; - - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime·throw("MHeapMap_Get"); - - p2 = m->p[i1]; - if(p2 == nil) - return nil; - return p2->s[i2]; -} - -void -runtime·MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s) -{ - int32 i1, i2; - - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime·throw("MHeapMap_Set"); - - m->p[i1]->s[i2] = s; -} - -// Allocate the storage required for entries [k, k+1, ..., k+len-1] -// so that Get and Set calls need not check for nil pointers. -bool -runtime·MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len) -{ - uintptr end; - int32 i1; - MHeapMapNode2 *p2; - - end = k+len; - while(k < end) { - if((k >> MHeapMap_TotalBits) != 0) - return false; - i1 = (k >> MHeapMap_Level2Bits) & MHeapMap_Level1Mask; - - // first-level pointer - if(m->p[i1] == nil) { - p2 = m->allocator(sizeof *p2); - if(p2 == nil) - return false; - mstats.heapmap_sys += sizeof *p2; - m->p[i1] = p2; - } - - // advance key past this leaf node - k = ((k >> MHeapMap_Level2Bits) + 1) << MHeapMap_Level2Bits; - } - return true; -} - diff --git a/src/pkg/runtime/mheapmap32.h b/src/pkg/runtime/mheapmap32.h deleted file mode 100644 index 29e619071..000000000 --- a/src/pkg/runtime/mheapmap32.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Free(v) must be able to determine the MSpan containing v. -// The MHeapMap is a 2-level radix tree mapping page numbers to MSpans. - -typedef struct MHeapMapNode2 MHeapMapNode2; - -enum -{ - // 32 bit address - 12 bit page size = 20 bits to map - MHeapMap_Level1Bits = 10, - MHeapMap_Level2Bits = 10, - - MHeapMap_TotalBits = - MHeapMap_Level1Bits + - MHeapMap_Level2Bits, - - MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1, - MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1, -}; - -struct MHeapMap -{ - void *(*allocator)(uintptr); - MHeapMapNode2 *p[1<<MHeapMap_Level1Bits]; -}; - -struct MHeapMapNode2 -{ - MSpan *s[1<<MHeapMap_Level2Bits]; -}; - -void runtime·MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)); -bool runtime·MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages); -MSpan* runtime·MHeapMap_Get(MHeapMap *m, PageID k); -MSpan* runtime·MHeapMap_GetMaybe(MHeapMap *m, PageID k); -void runtime·MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v); - - diff --git a/src/pkg/runtime/mheapmap32_defs.go b/src/pkg/runtime/mheapmap32_defs.go deleted file mode 100644 index 755725b46..000000000 --- a/src/pkg/runtime/mheapmap32_defs.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const ( - mHeapMap_Level1Bits = 10 - mHeapMap_Level2Bits = 10 - mHeapMap_TotalBits = mHeapMap_Level1Bits + mHeapMap_Level2Bits - - mHeapMap_Level1Mask = (1 << mHeapMap_Level1Bits) - 1 - mHeapMap_Level2Mask = (1 << mHeapMap_Level2Bits) - 1 -) - -type mHeapMap struct { - allocator func(uintptr) - p [1 << mHeapMap_Level1Bits]*mHeapMapNode2 -} - -type mHeapMapNode2 struct { - s [1 << mHeapMap_Level2Bits]*mSpan -} diff --git a/src/pkg/runtime/mheapmap64.c b/src/pkg/runtime/mheapmap64.c deleted file mode 100644 index e45ac9413..000000000 --- a/src/pkg/runtime/mheapmap64.c +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Heap map, 64-bit version -// See malloc.h and mheap.c for overview. - -#include "runtime.h" -#include "malloc.h" - -// 3-level radix tree mapping page ids to Span*. -void -runtime·MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)) -{ - m->allocator = allocator; -} - -MSpan* -runtime·MHeapMap_Get(MHeapMap *m, PageID k) -{ - int32 i1, i2, i3; - - i3 = k & MHeapMap_Level3Mask; - k >>= MHeapMap_Level3Bits; - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime·throw("MHeapMap_Get"); - - return m->p[i1]->p[i2]->s[i3]; -} - -MSpan* -runtime·MHeapMap_GetMaybe(MHeapMap *m, PageID k) -{ - int32 i1, i2, i3; - MHeapMapNode2 *p2; - MHeapMapNode3 *p3; - - i3 = k & MHeapMap_Level3Mask; - k >>= MHeapMap_Level3Bits; - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime·throw("MHeapMap_Get"); - - p2 = m->p[i1]; - if(p2 == nil) - return nil; - p3 = p2->p[i2]; - if(p3 == nil) - return nil; - return p3->s[i3]; -} - -void -runtime·MHeapMap_Set(MHeapMap *m, PageID k, MSpan *s) -{ - int32 i1, i2, i3; - - i3 = k & MHeapMap_Level3Mask; - k >>= MHeapMap_Level3Bits; - i2 = k & MHeapMap_Level2Mask; - k >>= MHeapMap_Level2Bits; - i1 = k & MHeapMap_Level1Mask; - k >>= MHeapMap_Level1Bits; - if(k != 0) - runtime·throw("MHeapMap_Set"); - - m->p[i1]->p[i2]->s[i3] = s; -} - -// Allocate the storage required for entries [k, k+1, ..., k+len-1] -// so that Get and Set calls need not check for nil pointers. -bool -runtime·MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr len) -{ - uintptr end; - int32 i1, i2; - MHeapMapNode2 *p2; - MHeapMapNode3 *p3; - - end = k+len; - while(k < end) { - if((k >> MHeapMap_TotalBits) != 0) - return false; - i2 = (k >> MHeapMap_Level3Bits) & MHeapMap_Level2Mask; - i1 = (k >> (MHeapMap_Level3Bits + MHeapMap_Level2Bits)) & MHeapMap_Level1Mask; - - // first-level pointer - if((p2 = m->p[i1]) == nil) { - p2 = m->allocator(sizeof *p2); - if(p2 == nil) - return false; - mstats.heapmap_sys += sizeof *p2; - m->p[i1] = p2; - } - - // second-level pointer - if(p2->p[i2] == nil) { - p3 = m->allocator(sizeof *p3); - if(p3 == nil) - return false; - mstats.heapmap_sys += sizeof *p3; - p2->p[i2] = p3; - } - - // advance key past this leaf node - k = ((k >> MHeapMap_Level3Bits) + 1) << MHeapMap_Level3Bits; - } - return true; -} - diff --git a/src/pkg/runtime/mheapmap64.h b/src/pkg/runtime/mheapmap64.h deleted file mode 100644 index a9934d2b1..000000000 --- a/src/pkg/runtime/mheapmap64.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Free(v) must be able to determine the MSpan containing v. -// The MHeapMap is a 3-level radix tree mapping page numbers to MSpans. -// -// NOTE(rsc): On a 32-bit platform (= 20-bit page numbers), -// we can swap in a 2-level radix tree. -// -// NOTE(rsc): We use a 3-level tree because tcmalloc does, but -// having only three levels requires approximately 1 MB per node -// in the tree, making the minimum map footprint 3 MB. -// Using a 4-level tree would cut the minimum footprint to 256 kB. -// On the other hand, it's just virtual address space: most of -// the memory is never going to be touched, thus never paged in. - -typedef struct MHeapMapNode2 MHeapMapNode2; -typedef struct MHeapMapNode3 MHeapMapNode3; - -enum -{ - // 64 bit address - 12 bit page size = 52 bits to map - MHeapMap_Level1Bits = 18, - MHeapMap_Level2Bits = 18, - MHeapMap_Level3Bits = 16, - - MHeapMap_TotalBits = - MHeapMap_Level1Bits + - MHeapMap_Level2Bits + - MHeapMap_Level3Bits, - - MHeapMap_Level1Mask = (1<<MHeapMap_Level1Bits) - 1, - MHeapMap_Level2Mask = (1<<MHeapMap_Level2Bits) - 1, - MHeapMap_Level3Mask = (1<<MHeapMap_Level3Bits) - 1, -}; - -struct MHeapMap -{ - void *(*allocator)(uintptr); - MHeapMapNode2 *p[1<<MHeapMap_Level1Bits]; -}; - -struct MHeapMapNode2 -{ - MHeapMapNode3 *p[1<<MHeapMap_Level2Bits]; -}; - -struct MHeapMapNode3 -{ - MSpan *s[1<<MHeapMap_Level3Bits]; -}; - -void runtime·MHeapMap_Init(MHeapMap *m, void *(*allocator)(uintptr)); -bool runtime·MHeapMap_Preallocate(MHeapMap *m, PageID k, uintptr npages); -MSpan* runtime·MHeapMap_Get(MHeapMap *m, PageID k); -MSpan* runtime·MHeapMap_GetMaybe(MHeapMap *m, PageID k); -void runtime·MHeapMap_Set(MHeapMap *m, PageID k, MSpan *v); - - diff --git a/src/pkg/runtime/mheapmap64_defs.go b/src/pkg/runtime/mheapmap64_defs.go deleted file mode 100644 index d7ba2b420..000000000 --- a/src/pkg/runtime/mheapmap64_defs.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -const ( - mHeapMap_Level1Bits = 18 - mHeapMap_Level2Bits = 18 - mHeapMap_Level3Bits = 16 - mHeapMap_TotalBits = mHeapMap_Level1Bits + mHeapMap_Level2Bits + mHeapMap_Level3Bits - - mHeapMap_Level1Mask = (1 << mHeapMap_Level1Bits) - 1 - mHeapMap_Level2Mask = (1 << mHeapMap_Level2Bits) - 1 - mHeapMap_Level3Mask = (1 << mHeapMap_Level3Bits) - 1 -) - -type mHeapMap struct { - allocator func(uintptr) - p [1 << mHeapMap_Level1Bits]*mHeapMapNode2 -} - - -type mHeapMapNode2 struct { - p [1 << mHeapMap_Level2Bits]*mHeapMapNode3 -} - - -type mHeapMapNode3 struct { - s [1 << mHeapMap_Level3Bits]*mSpan -} diff --git a/src/pkg/runtime/mkgodefs.sh b/src/pkg/runtime/mkgodefs.sh new file mode 100755 index 000000000..b6e97213e --- /dev/null +++ b/src/pkg/runtime/mkgodefs.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +cat <<EOF +// Go definitions for C variables and types. +// AUTOMATICALLY GENERATED BY THE FOLLOWING COMMAND. DO NOT EDIT. +// CC="$CC" CFLAGS="$CFLAGS" ./mkgodefs.sh $@ + +package runtime +import "unsafe" +var _ unsafe.Pointer + +EOF + +for i in "$@"; do + $CC $CFLAGS -q $i +done | awk ' +/^func/ { next } +/^const/ { next } +/^\/\/.*type/ { next } + +/^(const|func|type|var) / { + if(seen[$2]++) { + skip = /{[^}]*$/; + next; + } +} + +skip { + skip = !/^}/ + next; +} + +{print} +' diff --git a/src/pkg/runtime/plan9/runtime_defs.go b/src/pkg/runtime/plan9/runtime_defs.go deleted file mode 100644 index cf0b414a9..000000000 --- a/src/pkg/runtime/plan9/runtime_defs.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is runtime.h - -package runtime - -type lock struct { - key uint32 - sema uint32 -} - -type usema struct { - u uint32 - k uint32 -} - - -type note struct { - wakeup int32 - sema usema -} diff --git a/src/pkg/runtime/pprof/pprof.go b/src/pkg/runtime/pprof/pprof.go index d0cc73089..9bee51128 100644 --- a/src/pkg/runtime/pprof/pprof.go +++ b/src/pkg/runtime/pprof/pprof.go @@ -88,7 +88,6 @@ func WriteHeapProfile(w io.Writer) os.Error { fmt.Fprintf(b, "# Stack = %d / %d\n", s.StackInuse, s.StackSys) fmt.Fprintf(b, "# MSpan = %d / %d\n", s.MSpanInuse, s.MSpanSys) fmt.Fprintf(b, "# MCache = %d / %d\n", s.MCacheInuse, s.MCacheSys) - fmt.Fprintf(b, "# MHeapMapSys = %d\n", s.MHeapMapSys) fmt.Fprintf(b, "# BuckHashSys = %d\n", s.BuckHashSys) fmt.Fprintf(b, "# NextGC = %d\n", s.NextGC) diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index e9a19d950..998cbc7bc 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -470,8 +470,8 @@ scheduler(void) d = gp->defer; gp->defer = d->link; - // unwind to the stack frame with d->sp in it. - unwindstack(gp, d->sp); + // unwind to the stack frame with d's arguments in it. + unwindstack(gp, d->argp); // make the deferproc for this d return again, // this time returning 1. function will jump to @@ -481,7 +481,11 @@ scheduler(void) // each call to deferproc. // (the pc we're returning to does pop pop // before it tests the return value.) - gp->sched.sp = runtime·getcallersp(d->sp - 2*sizeof(uintptr)); + // on the arm there are 2 saved LRs mixed in too. + if(thechar == '5') + gp->sched.sp = (byte*)d->argp - 4*sizeof(uintptr); + else + gp->sched.sp = (byte*)d->argp - 2*sizeof(uintptr); gp->sched.pc = d->pc; gp->status = Grunning; runtime·free(d); @@ -633,7 +637,6 @@ void runtime·startcgocallback(G* g1) { Defer *d; - uintptr arg; runtime·lock(&runtime·sched); g1->status = Grunning; @@ -675,80 +678,11 @@ runtime·endcgocallback(G* g1) runtime·free(d); } -/* - * stack layout parameters. - * known to linkers. - * - * g->stackguard is set to point StackGuard bytes - * above the bottom of the stack. each function - * compares its stack pointer against g->stackguard - * to check for overflow. to cut one instruction from - * the check sequence for functions with tiny frames, - * the stack is allowed to protrude StackSmall bytes - * below the stack guard. functions with large frames - * don't bother with the check and always call morestack. - * the sequences are: - * - * guard = g->stackguard - * frame = function's stack frame size - * argsize = size of function arguments (call + return) - * - * stack frame size <= StackSmall: - * CMPQ guard, SP - * JHI 3(PC) - * MOVQ m->morearg, $(argsize << 32) - * CALL sys.morestack(SB) - * - * stack frame size > StackSmall but < StackBig - * LEAQ (frame-StackSmall)(SP), R0 - * CMPQ guard, R0 - * JHI 3(PC) - * MOVQ m->morearg, $(argsize << 32) - * CALL sys.morestack(SB) - * - * stack frame size >= StackBig: - * MOVQ m->morearg, $((argsize << 32) | frame) - * CALL sys.morestack(SB) - * - * the bottom StackGuard - StackSmall bytes are important: - * there has to be enough room to execute functions that - * refuse to check for stack overflow, either because they - * need to be adjacent to the actual caller's frame (sys.deferproc) - * or because they handle the imminent stack overflow (sys.morestack). - * - * for example, sys.deferproc might call malloc, - * which does one of the above checks (without allocating a full frame), - * which might trigger a call to sys.morestack. - * this sequence needs to fit in the bottom section of the stack. - * on amd64, sys.morestack's frame is 40 bytes, and - * sys.deferproc's frame is 56 bytes. that fits well within - * the StackGuard - StackSmall = 128 bytes at the bottom. - * there may be other sequences lurking or yet to be written - * that require more stack. sys.morestack checks to make sure - * the stack has not completely overflowed and should - * catch such sequences. - */ -enum -{ - // byte offset of stack guard (g->stackguard) above bottom of stack. - StackGuard = 256, - - // checked frames are allowed to protrude below the guard by - // this many bytes. this saves an instruction in the checking - // sequence when the stack frame is tiny. - StackSmall = 128, - - // extra space in the frame (beyond the function for which - // the frame is allocated) is assumed not to be much bigger - // than this amount. it may not be used efficiently if it is. - StackBig = 4096, -}; - void runtime·oldstack(void) { Stktop *top, old; - uint32 args; + uint32 argsize; byte *sp; G *g1; static int32 goid; @@ -759,15 +693,15 @@ runtime·oldstack(void) top = (Stktop*)g1->stackbase; sp = (byte*)top; old = *top; - args = old.args; - if(args > 0) { - sp -= args; - runtime·mcpy(top->fp, sp, args); + argsize = old.argsize; + if(argsize > 0) { + sp -= argsize; + runtime·mcpy(top->argp, sp, argsize); } goid = old.gobuf.g->goid; // fault if g is bad, before gogo - if(old.free) - runtime·stackfree(g1->stackguard - StackGuard); + if(old.free != 0) + runtime·stackfree(g1->stackguard - StackGuard, old.free); g1->stackbase = old.stackbase; g1->stackguard = old.stackguard; @@ -777,40 +711,45 @@ runtime·oldstack(void) void runtime·newstack(void) { - int32 frame, args; + int32 framesize, argsize; Stktop *top; byte *stk, *sp; G *g1; Gobuf label; - bool free; + bool reflectcall; + uintptr free; - frame = m->moreframe; - args = m->moreargs; + framesize = m->moreframesize; + argsize = m->moreargsize; g1 = m->curg; - if(m->morebuf.sp < g1->stackguard - StackGuard) - runtime·throw("split stack overflow"); + if(m->morebuf.sp < g1->stackguard - StackGuard) { + runtime·printf("runtime: split stack overflow: %p < %p\n", m->morebuf.sp, g1->stackguard - StackGuard); + runtime·throw("runtime: split stack overflow"); + } + + reflectcall = framesize==1; + if(reflectcall) + framesize = 0; - if(frame == 1 && args > 0 && m->morebuf.sp - sizeof(Stktop) - args - 32 > g1->stackguard) { - // special case: called from reflect.call (frame == 1) + if(reflectcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > g1->stackguard) { + // special case: called from reflect.call (framesize==1) // to call code with an arbitrary argument size, // and we have enough space on the current stack. // the new Stktop* is necessary to unwind, but // we don't need to create a new segment. top = (Stktop*)(m->morebuf.sp - sizeof(*top)); stk = g1->stackguard - StackGuard; - free = false; + free = 0; } else { // allocate new segment. - if(frame == 1) // failed reflect.call hint - frame = 0; - frame += args; - if(frame < StackBig) - frame = StackBig; - frame += 1024; // room for more functions, Stktop. - stk = runtime·stackalloc(frame); - top = (Stktop*)(stk+frame-sizeof(*top)); - free = true; + framesize += argsize; + if(framesize < StackBig) + framesize = StackBig; + framesize += StackExtra; // room for more functions, Stktop. + stk = runtime·stackalloc(framesize); + top = (Stktop*)(stk+framesize-sizeof(*top)); + free = framesize; } //printf("newstack frame=%d args=%d morepc=%p morefp=%p gobuf=%p, %p newstk=%p\n", @@ -819,8 +758,8 @@ runtime·newstack(void) top->stackbase = g1->stackbase; top->stackguard = g1->stackguard; top->gobuf = m->morebuf; - top->fp = m->morefp; - top->args = args; + top->argp = m->moreargp; + top->argsize = argsize; top->free = free; // copy flag from panic @@ -831,9 +770,14 @@ runtime·newstack(void) g1->stackguard = stk + StackGuard; sp = (byte*)top; - if(args > 0) { - sp -= args; - runtime·mcpy(sp, m->morefp, args); + if(argsize > 0) { + sp -= argsize; + runtime·mcpy(sp, m->moreargp, argsize); + } + if(thechar == '5') { + // caller would have saved its LR below args. + sp -= sizeof(void*); + *(void**)sp = nil; } // Continue as if lessstack had just called m->morepc @@ -876,7 +820,13 @@ runtime·malg(int32 stacksize) void runtime·newproc(int32 siz, byte* fn, ...) { - runtime·newproc1(fn, (byte*)(&fn+1), siz, 0); + byte *argp; + + if(thechar == '5') + argp = (byte*)(&fn+2); // skip caller's saved LR + else + argp = (byte*)(&fn+1); + runtime·newproc1(fn, argp, siz, 0); } G* @@ -899,7 +849,7 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret) if(newg->stackguard - StackGuard != newg->stack0) runtime·throw("invalid stack in newg"); } else { - newg = runtime·malg(4096); + newg = runtime·malg(StackBig); newg->status = Gwaiting; newg->alllink = runtime·allg; runtime·allg = newg; @@ -908,6 +858,11 @@ runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret) sp = newg->stackbase; sp -= siz; runtime·mcpy(sp, argp, narg); + if(thechar == '5') { + // caller's LR + sp -= sizeof(void*); + *(void**)sp = nil; + } newg->sched.sp = sp; newg->sched.pc = (byte*)runtime·goexit; @@ -933,10 +888,13 @@ runtime·deferproc(int32 siz, byte* fn, ...) d = runtime·malloc(sizeof(*d) + siz - sizeof(d->args)); d->fn = fn; - d->sp = (byte*)(&fn+1); d->siz = siz; d->pc = runtime·getcallerpc(&siz); - runtime·mcpy(d->args, d->sp, d->siz); + if(thechar == '5') + d->argp = (byte*)(&fn+2); // skip caller's saved link register + else + d->argp = (byte*)(&fn+1); + runtime·mcpy(d->args, d->argp, d->siz); d->link = g->defer; g->defer = d; @@ -955,19 +913,19 @@ void runtime·deferreturn(uintptr arg0) { Defer *d; - byte *sp, *fn; + byte *argp, *fn; d = g->defer; if(d == nil) return; - sp = runtime·getcallersp(&arg0); - if(d->sp != sp) + argp = (byte*)&arg0; + if(d->argp != argp) return; - runtime·mcpy(d->sp, d->args, d->siz); + runtime·mcpy(argp, d->args, d->siz); g->defer = d->link; fn = d->fn; runtime·free(d); - runtime·jmpdefer(fn, sp); + runtime·jmpdefer(fn, argp); } static void @@ -983,7 +941,7 @@ rundefer(void) } // Free stack frames until we hit the last one -// or until we find the one that contains the sp. +// or until we find the one that contains the argp. static void unwindstack(G *gp, byte *sp) { @@ -1000,8 +958,8 @@ unwindstack(G *gp, byte *sp) break; gp->stackbase = top->stackbase; gp->stackguard = top->stackguard; - if(top->free) - runtime·stackfree(stk); + if(top->free != 0) + runtime·stackfree(stk, top->free); } if(sp != nil && (sp < gp->stackguard - StackGuard || gp->stackbase < sp)) { @@ -1043,12 +1001,11 @@ runtime·panic(Eface e) // take defer off list in case of recursive panic g->defer = d->link; g->ispanic = true; // rock for newstack, where reflect.call ends up - if(thechar == '5') - reflect·call(d->fn, d->args+4, d->siz-4); // reflect.call does not expect LR - else - reflect·call(d->fn, d->args, d->siz); + reflect·call(d->fn, d->args, d->siz); if(p->recovered) { g->panic = p->link; + if(g->panic == nil) // must be done with signal + g->sig = 0; runtime·free(p); // put recovering defer back on list // for scheduler to find. @@ -1068,13 +1025,11 @@ runtime·panic(Eface e) #pragma textflag 7 /* no split, or else g->stackguard is not the stack for fp */ void -runtime·recover(byte *fp, Eface ret) +runtime·recover(byte *argp, Eface ret) { Stktop *top, *oldtop; Panic *p; - fp = runtime·getcallersp(fp); - // Must be a panic going on. if((p = g->panic) == nil || p->recovered) goto nomatch; @@ -1097,11 +1052,11 @@ runtime·recover(byte *fp, Eface ret) // allocated a second segment (see below), // the fp is slightly above top - top->args. // That condition can't happen normally though - // (stack pointer go down, not up), so we can accept + // (stack pointers go down, not up), so we can accept // any fp between top and top - top->args as // indicating the top of the segment. top = (Stktop*)g->stackbase; - if(fp < (byte*)top - top->args || (byte*)top < fp) + if(argp < (byte*)top - top->argsize || (byte*)top < argp) goto nomatch; // The deferred call makes a new segment big enough @@ -1117,7 +1072,7 @@ runtime·recover(byte *fp, Eface ret) // bytes above top->fp) abuts the old top of stack. // This is a correct test for both closure and non-closure code. oldtop = (Stktop*)top->stackbase; - if(oldtop != nil && top->fp == (byte*)oldtop - top->args) + if(oldtop != nil && top->argp == (byte*)oldtop - top->argsize) top = oldtop; // Now we have the segment that was created to @@ -1237,3 +1192,9 @@ runtime·Goroutines(int32 ret) ret = runtime·sched.gcount; FLUSH(&ret); } + +int32 +runtime·mcount(void) +{ + return runtime·sched.mcount; +} diff --git a/src/pkg/runtime/reflect.goc b/src/pkg/runtime/reflect.goc index a2e3c6ee1..71d648266 100644 --- a/src/pkg/runtime/reflect.goc +++ b/src/pkg/runtime/reflect.goc @@ -75,7 +75,7 @@ func chansend(ch *byte, val *byte, pres *bool) { } func chanrecv(ch *byte, val *byte, pres *bool) { - runtime·chanrecv((Hchan*)ch, val, pres); + runtime·chanrecv((Hchan*)ch, val, pres, nil); } func chanclose(ch *byte) { diff --git a/src/pkg/runtime/runtime-gdb.py b/src/pkg/runtime/runtime-gdb.py index a7ca94cdb..677e9bde4 100644 --- a/src/pkg/runtime/runtime-gdb.py +++ b/src/pkg/runtime/runtime-gdb.py @@ -161,7 +161,7 @@ def is_iface(val): def is_eface(val): try: - return str(val['type_'].type) == "runtime.Type *" \ + return str(val['_type'].type) == "struct runtime._type *" \ and str(val['data'].type) == "void *" except: pass @@ -185,14 +185,14 @@ def iface_dtype(obj): "Decode type of the data field of an eface or iface struct." if is_iface(obj): - go_type_ptr = obj['tab']['Type'] + go_type_ptr = obj['tab']['_type'] elif is_eface(obj): - go_type_ptr = obj['type_'] + go_type_ptr = obj['_type'] else: return ct = gdb.lookup_type("struct runtime.commonType").pointer() - dynamic_go_type = go_type_ptr['data'].cast(ct).dereference() + dynamic_go_type = go_type_ptr['ptr'].cast(ct).dereference() dtype_name = dynamic_go_type['string'].dereference()['str'].string() type_size = int(dynamic_go_type['size']) uintptr_size = int(dynamic_go_type['size'].type.sizeof) # size is itself an uintptr diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c index 9d3efe966..284b1e458 100644 --- a/src/pkg/runtime/runtime.c +++ b/src/pkg/runtime/runtime.c @@ -30,11 +30,16 @@ runtime·dopanic(int32 unused) } runtime·panicking++; - runtime·printf("\npanic PC=%X\n", (uint64)(uintptr)&unused); + if(g->sig != 0) + runtime·printf("\n[signal %x code=%p addr=%p pc=%p]\n", + g->sig, g->sigcode0, g->sigcode1, g->sigpc); + + runtime·printf("\n"); if(runtime·gotraceback()){ runtime·traceback(runtime·getcallerpc(&unused), runtime·getcallersp(&unused), 0, g); runtime·tracebackothers(g); } + runtime·breakpoint(); // so we can grab it in a debugger runtime·exit(2); } @@ -79,6 +84,10 @@ runtime·panicstring(int8 *s) { Eface err; + if(m->gcing) { + runtime·printf("panic: %s\n", s); + runtime·throw("panic during gc"); + } runtime·newErrorString(runtime·gostringnocopy((byte*)s), &err); runtime·panic(err); } @@ -148,6 +157,7 @@ runtime·args(int32 c, uint8 **v) } int32 runtime·isplan9; +int32 runtime·iswindows; void runtime·goargs(void) diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index bde62833e..2c19f851e 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -63,7 +63,7 @@ typedef struct Eface Eface; typedef struct Type Type; typedef struct Defer Defer; typedef struct Panic Panic; -typedef struct hash Hmap; +typedef struct Hmap Hmap; typedef struct Hchan Hchan; typedef struct Complex64 Complex64; typedef struct Complex128 Complex128; @@ -199,18 +199,19 @@ struct G int32 sig; uintptr sigcode0; uintptr sigcode1; + uintptr sigpc; }; struct M { // The offsets of these fields are known to (hard-coded in) libmach. G* g0; // goroutine with scheduling stack void (*morepc)(void); - void* morefp; // frame pointer for more stack + void* moreargp; // argument pointer for more stack Gobuf morebuf; // gobuf arg to morestack // Fields not known to debuggers. - uint32 moreframe; // size arguments to morestack - uint32 moreargs; + uint32 moreframesize; // size arguments to morestack + uint32 moreargsize; uintptr cret; // return value from C uint64 procid; // for debuggers, but offset not hard-coded G* gsignal; // signal-handling G @@ -234,7 +235,7 @@ struct M uint32 freghi[16]; // D[i] msb and F[i+16] uint32 fflag; // floating point compare flags #ifdef __WINDOWS__ - void* gostack; // bookmark to keep track of go stack during stdcall + void* sehframe; #endif }; struct Stktop @@ -243,13 +244,10 @@ struct Stktop uint8* stackguard; uint8* stackbase; Gobuf gobuf; - uint32 args; + uint32 argsize; - // Frame pointer: where args start in old frame. - // fp == gobuf.sp except in the case of a reflected - // function call, which uses an off-stack argument frame. - uint8* fp; - bool free; // call stackfree for this frame? + uint8* argp; // pointer to arguments in old frame + uintptr free; // if free>0, call stackfree using free as size bool panic; // is this frame the top of a panic? }; struct Alg @@ -333,7 +331,7 @@ enum { struct Defer { int32 siz; - byte* sp; + byte* argp; // where args were copied from byte* pc; byte* fn; Defer* link; @@ -423,7 +421,7 @@ void runtime·minit(void); Func* runtime·findfunc(uintptr); int32 runtime·funcline(Func*, uint64); void* runtime·stackalloc(uint32); -void runtime·stackfree(void*); +void runtime·stackfree(void*, uintptr); MCache* runtime·allocmcache(void); void runtime·mallocinit(void); bool runtime·ifaceeq_c(Iface, Iface); @@ -438,13 +436,14 @@ void runtime·addfinalizer(void*, void(*fn)(void*), int32); void runtime·walkfintab(void (*fn)(void*)); void runtime·runpanic(Panic*); void* runtime·getcallersp(void*); +int32 runtime·mcount(void); void runtime·exit(int32); void runtime·breakpoint(void); void runtime·gosched(void); void runtime·goexit(void); void runtime·runcgo(void (*fn)(void*), void*); -void runtime·runcgocallback(G*, void*, void (*fn)()); +uintptr runtime·runcgocallback(G*, void*, void (*fn)()); void runtime·entersyscall(void); void runtime·exitsyscall(void); void runtime·startcgocallback(G*); @@ -508,11 +507,11 @@ void runtime·notewakeup(Note*); #define EACCES 13 /* - * low level go-called + * low level C-called */ uint8* runtime·mmap(byte*, uintptr, int32, int32, int32, uint32); void runtime·munmap(uint8*, uintptr); -void runtime·memclr(byte*, uint32); +void runtime·memclr(byte*, uintptr); void runtime·setcallerpc(void*, void*); void* runtime·getcallerpc(void*); @@ -583,10 +582,91 @@ Hmap* runtime·makemap_c(Type*, Type*, int64); Hchan* runtime·makechan_c(Type*, int64); void runtime·chansend(Hchan*, void*, bool*); -void runtime·chanrecv(Hchan*, void*, bool*); +void runtime·chanrecv(Hchan*, void*, bool*, bool*); void runtime·chanclose(Hchan*); bool runtime·chanclosed(Hchan*); int32 runtime·chanlen(Hchan*); int32 runtime·chancap(Hchan*); void runtime·ifaceE2I(struct InterfaceType*, Eface, Iface*); + +/* + * Stack layout parameters. + * Known to linkers. + * + * The per-goroutine g->stackguard is set to point + * StackGuard bytes above the bottom of the stack. + * Each function compares its stack pointer against + * g->stackguard to check for overflow. To cut one + * instruction from the check sequence for functions + * with tiny frames, the stack is allowed to protrude + * StackSmall bytes below the stack guard. Functions + * with large frames don't bother with the check and + * always call morestack. The sequences are + * (for amd64, others are similar): + * + * guard = g->stackguard + * frame = function's stack frame size + * argsize = size of function arguments (call + return) + * + * stack frame size <= StackSmall: + * CMPQ guard, SP + * JHI 3(PC) + * MOVQ m->morearg, $(argsize << 32) + * CALL morestack(SB) + * + * stack frame size > StackSmall but < StackBig + * LEAQ (frame-StackSmall)(SP), R0 + * CMPQ guard, R0 + * JHI 3(PC) + * MOVQ m->morearg, $(argsize << 32) + * CALL morestack(SB) + * + * stack frame size >= StackBig: + * MOVQ m->morearg, $((argsize << 32) | frame) + * CALL morestack(SB) + * + * The bottom StackGuard - StackSmall bytes are important: + * there has to be enough room to execute functions that + * refuse to check for stack overflow, either because they + * need to be adjacent to the actual caller's frame (deferproc) + * or because they handle the imminent stack overflow (morestack). + * + * For example, deferproc might call malloc, which does one + * of the above checks (without allocating a full frame), + * which might trigger a call to morestack. This sequence + * needs to fit in the bottom section of the stack. On amd64, + * morestack's frame is 40 bytes, and deferproc's frame is 56 bytes. + * That fits well within the StackGuard - StackSmall = 128 bytes + * at the bottom. There may be other sequences lurking or yet to + * be written that require more stack. Morestack checks to make + * sure the stack has not completely overflowed and should catch + * such sequences. + */ +enum +{ +#ifdef __WINDOWS__ + // need enough room in guard area for exception handler. + // use larger stacks to compensate for larger stack guard. + StackSmall = 256, + StackGuard = 2048, + StackBig = 8192, + StackExtra = StackGuard, +#else + // byte offset of stack guard (g->stackguard) above bottom of stack. + StackGuard = 256, + + // checked frames are allowed to protrude below the guard by + // this many bytes. this saves an instruction in the checking + // sequence when the stack frame is tiny. + StackSmall = 128, + + // extra space in the frame (beyond the function for which + // the frame is allocated) is assumed not to be much bigger + // than this amount. it may not be used efficiently if it is. + StackBig = 4096, + + // extra room over frame size when allocating a stack. + StackExtra = 1024, +#endif +}; diff --git a/src/pkg/runtime/runtime_defs.go b/src/pkg/runtime/runtime_defs.go deleted file mode 100644 index ba3c3ed75..000000000 --- a/src/pkg/runtime/runtime_defs.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is runtime.h - -// TODO(lvd): automate conversion to all the _defs.go files - -package runtime - -import "unsafe" - -const ( - gidle = iota - grunnable - grunning - gsyscall - gwaiting - gmoribund - gdead - grecovery -) - -// const ( Structrnd = sizeof(uintptr) ) - -type string_ struct { - str *byte - len int32 -} - -type iface struct { - tab *itab - data unsafe.Pointer -} - -type eface struct { - type_ *Type - data unsafe.Pointer -} - -type complex64 struct { - real float32 - imag float32 -} - -type complex128 struct { - real float64 - imag float64 -} - -type slice struct { - array *byte - len uint32 - cap uint32 -} - -type gobuf struct { - sp unsafe.Pointer - pc unsafe.Pointer - g *g_ -} - -type g_ struct { - stackguard unsafe.Pointer - stackbase unsafe.Pointer - defer_ *defer_ - panic_ *panic_ - sched gobuf - stack0 unsafe.Pointer - entry unsafe.Pointer - alllink *g_ - param unsafe.Pointer - status int16 - goid int32 - selgen uint32 - schedlink *g_ - readyonstop bool - ispanic bool - m *m_ - lockedm *m_ - sig int32 - sigcode0 uintptr - sigcode1 uintptr -} - -type m_ struct { - g0 *g_ - morepc unsafe.Pointer - morefp unsafe.Pointer - morebuf gobuf - moreframe uint32 - moreargs uint32 - cret uintptr - procid uint64 - gsignal *g_ - tls [8]uint32 - sched gobuf - curg *g_ - id int32 - mallocing int32 - gcing int32 - locks int32 - nomemprof int32 - waitnextg int32 - havenextg note - nextg *g_ - alllink *m_ - schedlink *m_ - machport uint32 - mcache *mCache - lockedg *g_ - freg [8]uint64 - // gostack unsafe.Pointer // __WINDOWS__ -} - -type stktop struct { - stackguard *uint8 - stackbase *uint8 - gobuf gobuf - args uint32 - fp *uint8 - free bool - panic_ bool -} - -type alg struct { - hash func(uint32, unsafe.Pointer) uintptr - equal func(uint32, unsafe.Pointer, unsafe.Pointer) uint32 - print func(uint32, unsafe.Pointer) - copy func(uint32, unsafe.Pointer, unsafe.Pointer) -} - -type sigtab struct { - flags int32 - name *int8 -} - -const ( - sigCatch = (1 << iota) - sigIgnore - sigRestart - sigQueue - sigPanic -) - -type Func struct { - name string - typ string - src string - pcln []byte - entry uintptr - pc0 uintptr - ln0 int32 - frame int32 - args int32 - locals int32 -} - -const ( - aMEM = iota - aNOEQ - aSTRING - aINTER - aNILINTER - aMEMWORD - amax -) - -type defer_ struct { - siz int32 - sp unsafe.Pointer - pc unsafe.Pointer - fn unsafe.Pointer - link *defer_ - args [8]byte // padded to actual size -} - -type panic_ struct { - arg eface - stackbase unsafe.Pointer - link *panic_ - recovered bool -} - -/* - * External data. - */ - -var ( - algarray [amax]alg - emptystring string - allg *g_ - allm *m_ - goidgen int32 - gomaxprocs int32 - panicking int32 - fd int32 - gcwaiting int32 - goos *int8 -) diff --git a/src/pkg/runtime/tiny/386/defs.h b/src/pkg/runtime/tiny/386/defs.h deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/386/defs.h +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/386/rt0.s b/src/pkg/runtime/tiny/386/rt0.s deleted file mode 100644 index 524ac7664..000000000 --- a/src/pkg/runtime/tiny/386/rt0.s +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -TEXT _rt0_386_tiny(SB), 7, $0 - // Disable interrupts. - CLI - - // Establish stack. - MOVL $0x10000, AX - MOVL AX, SP - - // Set up memory hardware. - CALL runtime·msetup(SB) - - // _rt0_386 expects to find argc, argv, envv on stack. - // Set up argv=["kernel"] and envv=[]. - SUBL $64, SP - MOVL $1, 0(SP) - MOVL $runtime·kernel(SB), 4(SP) - MOVL $0, 8(SP) - MOVL $0, 12(SP) - JMP _rt0_386(SB) - -DATA runtime·kernel(SB)/7, $"kernel\z" -GLOBL runtime·kernel(SB), $7 - - diff --git a/src/pkg/runtime/tiny/386/signal.c b/src/pkg/runtime/tiny/386/signal.c deleted file mode 100644 index 88e634e9d..000000000 --- a/src/pkg/runtime/tiny/386/signal.c +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" - -extern void runtime·write(int32 fd, void *v, int32 len, int32 cap); // slice, spelled out - -int32 -runtime·write(int32 fd, void *v, int32 len) -{ - runtime·write(fd, v, len, len); - return len; -} - -void -runtime·gettime(int64*, int32*) -{ -} diff --git a/src/pkg/runtime/tiny/386/sys.s b/src/pkg/runtime/tiny/386/sys.s deleted file mode 100644 index 851171476..000000000 --- a/src/pkg/runtime/tiny/386/sys.s +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Manipulation of segment tables. -// -// Descriptor entry format for system call -// is the native machine format, ugly as it is: -// -// 2-byte limit -// 3-byte base -// 1-byte: 0x80=present, 0x60=dpl<<5, 0x1F=type -// 1-byte: 0x80=limit is *4k, 0x40=32-bit operand size, -// 0x0F=4 more bits of limit -// 1 byte: 8 more bits of base - -// Called to set up memory hardware. -// Already running in 32-bit mode thanks to boot block, -// but we need to install our new GDT that we can modify. -TEXT runtime·msetup(SB), 7, $0 - MOVL runtime·gdtptr(SB), GDTR - MOVL $(1*8+0), AX - MOVW AX, DS - MOVW AX, ES - MOVW AX, SS - MOVW $0, AX - MOVW AX, FS - MOVW AX, GS - - // long jmp to cs:mret - BYTE $0xEA - LONG $runtime·mret(SB) - WORD $(2*8+0) - -TEXT runtime·mret(SB), 7, $0 - RET - -// GDT memory -TEXT runtime·gdt(SB), 7, $0 - // null segment - LONG $0 - LONG $0 - - // 4GB data segment - LONG $0xFFFF - LONG $0x00CF9200 - - // 4GB code segment - LONG $0xFFFF - LONG $0x00CF9A00 - - // null segment (will be thread-local storage segment) - LONG $0 - LONG $0 - -// GDT pseudo-descriptor -TEXT runtime·gdtptr(SB), 7, $0 - WORD $(4*8) - LONG $runtime·gdt(SB) - -// Called to establish the per-thread segment. -// Write to gdt[3] and reload the gdt register. -// setldt(int entry, int address, int limit) -TEXT runtime·setldt(SB),7,$32 - MOVL address+4(FP), BX // aka base - MOVL limit+8(FP), CX - - // set up segment descriptor - LEAL gdt+(3*8)(SB), AX // gdt entry #3 - MOVL $0, 0(AX) - MOVL $0, 4(AX) - - MOVW BX, 2(AX) - SHRL $16, BX - MOVB BX, 4(AX) - SHRL $8, BX - MOVB BX, 7(AX) - - MOVW CX, 0(AX) - SHRL $16, CX - ANDL $0x0F, CX - ORL $0x40, CX // 32-bit operand size - MOVB CX, 6(AX) - MOVB $0xF2, 5(AX) // r/w data descriptor, dpl=3, present - - MOVL runtime·gdtptr(SB), GDTR - - // Compute segment selector - (entry*8+0) - MOVL $(3*8+0), AX - MOVW AX, GS - RET - diff --git a/src/pkg/runtime/tiny/README b/src/pkg/runtime/tiny/README deleted file mode 100755 index cf001d1e6..000000000 --- a/src/pkg/runtime/tiny/README +++ /dev/null @@ -1,123 +0,0 @@ -This directory contains a simple example of how one might -start Go running on bare hardware. There is currently code -for 386 and arm. - - -386 - -It is very primitive but can run go/test/sieve.go, the concurrent -prime sieve, on a uniprocessor. - -To run, first build the tools by running all.bash with GOARCH=386 -and GOOS set to your normal GOOS (linux, darwin). Then: - - export GOOS=tiny - cd $GOROOT/src/pkg/runtime - make clean - make install - cd tiny - 8g $GOROOT/test/sieve.go - 8l sieve.8 - 8l -a sieve.8 >sieve.asm # can consult sieve.asm for debugging - dd if=/dev/zero of=disk count=10000 - cat bootblock 8.out | dd of=disk conv=notrunc - -Use the built-in print(text string) function to print to the -console. - - -BOCHS - -You may have to tweak the .bochsrc depending on your system, -and you may need to install the Bochs emulator. - - $ cp dot-bochsrc .bochsrc - $ $EDITOR .bochsrc # tweak it if required - $ bochs - - -ORACLE xVM VIRTUALBOX - -Install VirtualBox. Then: - - Build 'disk' (described above under '386'). - - $ VBoxManage convertfromraw disk go-tiny.vdi - $ VirtualBox - create a new VM; as disk use the go-tiny.vdi image. - start the vm. - - -QEMU / KVM - -This should work the same for qemu and kvm (really: qemu-kvm). - - Build 'disk' (described above under '386'). - - $ qemu -hda disk - - -ARM - -First build the toolchain using GOARCH=arm and GOOS=linux. When -you build your embedded code set GOARCH=tiny. - - export GOOS=tiny - cd $GOROOT/src/pkg/runtime - make clean - make install - -On arm the tiny runtime doesn't define a low level write function. You can either -define a stub if you don't need debug output, or more usefully, define it to -print to some debug serial port. Here is a sample function that prints to -the DBGU on an at91sam7s: - -#define DBGU_CSR ((uint32*) 0xFFFFF214) // (DBGU) Channel Status Register -#define US_TXRDY ((uint32) 0x1 << 1) // (DBGU) TXRDY Interrupt -#define DBGU_THR ((uint32*) 0xFFFFF21C) // (DBGU) Transmitter Holding Register - -int32 -write(int32 fd, void* b, int32 n) -{ - uint32 i; - uint8* s = (uint8*)b; - - for (i = 0; i < n; i++) { - while ((*DBGU_CSR & US_TXRDY) == 0) { - } - *DBGU_THR = *s; - s++; - } - return n; -} - - - -The 386 bootblock is from MIT's xv6 project and carries this notice: - - The xv6 software is: - - Copyright (c) 2006-2009 Frans Kaashoek, Robert Morris, Russ Cox, - Massachusetts Institute of Technology - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -http://pdos.csail.mit.edu/6.828/xv6/ - diff --git a/src/pkg/runtime/tiny/arm/defs.h b/src/pkg/runtime/tiny/arm/defs.h deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/arm/defs.h +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/arm/rt0.s b/src/pkg/runtime/tiny/arm/rt0.s deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/arm/rt0.s +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/arm/signal.c b/src/pkg/runtime/tiny/arm/signal.c deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/arm/signal.c +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/arm/sys.s b/src/pkg/runtime/tiny/arm/sys.s deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/arm/sys.s +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/bootblock b/src/pkg/runtime/tiny/bootblock Binary files differdeleted file mode 100755 index 54dd7c632..000000000 --- a/src/pkg/runtime/tiny/bootblock +++ /dev/null diff --git a/src/pkg/runtime/tiny/dot-bochsrc b/src/pkg/runtime/tiny/dot-bochsrc deleted file mode 100644 index 3f5c813a2..000000000 --- a/src/pkg/runtime/tiny/dot-bochsrc +++ /dev/null @@ -1,18 +0,0 @@ -romimage: file=$BXSHARE/BIOS-bochs-latest -cpu: count=1, ips=100000000, reset_on_triple_fault=0 -megs: 32 -vgaromimage: file=/usr/share/vgabios/vgabios.bin -vga: extension=none -ata0: enabled=1, ioaddr1=0x1f0, ioaddr2=0x3f0, irq=14 -ata1: enabled=1, ioaddr1=0x170, ioaddr2=0x370, irq=15 -ata2: enabled=0, ioaddr1=0x1e8, ioaddr2=0x3e0, irq=11 -ata3: enabled=0, ioaddr1=0x168, ioaddr2=0x360, irq=9 -ata0-master: type=disk, mode=flat, path="disk", cylinders=100, heads=10, spt=10 -boot: disk -panic: action=ask -error: action=report -info: action=report -debug: action=ignore -debugger_log: - -config_interface: wx -display_library: wx diff --git a/src/pkg/runtime/tiny/io.go b/src/pkg/runtime/tiny/io.go deleted file mode 100644 index f30e68889..000000000 --- a/src/pkg/runtime/tiny/io.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Simple CGA screen output - -package runtime - -import "unsafe" - -var crt *[25 * 80]uint16 -var pos int - -func putc(c int) { - const ( - port = 0x3d4 - color = 0x0700 // white on black - ) - - if crt == nil { - // init on demand in case printf is called before - // initialization runs. - var mem uintptr = 0xb8000 - crt = (*[25 * 80]uint16)(unsafe.Pointer(mem)) - pos = 0 - for i := range crt[0:] { - crt[i] = 0 - } - } - - switch c { - case '\n': - pos += 80 - pos%80 - default: - crt[pos] = uint16(c&0xff | color) - pos++ - } - - if pos/80 >= 24 { - copy(crt[0:], crt[80:]) - pos -= 80 - for i := 0; i < 80; i++ { - crt[24*80+i] = 0 - } - } - crt[pos] = ' ' | color -} - -func write(fd int32, b []byte) { - for _, c := range b { - putc(int(c)) - } -} diff --git a/src/pkg/runtime/tiny/mem.c b/src/pkg/runtime/tiny/mem.c deleted file mode 100644 index 7abecfba0..000000000 --- a/src/pkg/runtime/tiny/mem.c +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "malloc.h" - -// Assume there's an arbitrary amount of memory starting at "end". -// Sizing PC memory is beyond the scope of this demo. - -static byte *allocp; - -void* -runtime·SysAlloc(uintptr ask) -{ - extern byte end[]; - byte *q; - - if(allocp == nil) { - allocp = end; - allocp += 7 & -(uintptr)allocp; - } - ask += 7 & -ask; - - q = allocp; - allocp += ask; - runtime·memclr(q, ask); - return q; -} - -void -runtime·SysFree(void *v, uintptr n) -{ - // Push pointer back if this is a free - // of the most recent SysAlloc. - n += 7 & -n; - if(allocp == (byte*)v+n) - allocp -= n; -} - -void -runtime·SysUnused(void *v, uintptr n) -{ - USED(v, n); -} - -void -runtime·SysMemInit(void) -{ -} diff --git a/src/pkg/runtime/tiny/os.h b/src/pkg/runtime/tiny/os.h deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/os.h +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/runtime_defs.go b/src/pkg/runtime/tiny/runtime_defs.go deleted file mode 100644 index 86de13316..000000000 --- a/src/pkg/runtime/tiny/runtime_defs.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// OS-Specific Go definitions of internal structures. Master is runtime.h - -package runtime - -type lock struct { - key uint32 - sema uint32 -} - -type note lock diff --git a/src/pkg/runtime/tiny/signals.h b/src/pkg/runtime/tiny/signals.h deleted file mode 100644 index 5df757613..000000000 --- a/src/pkg/runtime/tiny/signals.h +++ /dev/null @@ -1 +0,0 @@ -// nothing to see here diff --git a/src/pkg/runtime/tiny/thread.c b/src/pkg/runtime/tiny/thread.c deleted file mode 100644 index 0572ecb77..000000000 --- a/src/pkg/runtime/tiny/thread.c +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" - -int8 *goos = "tiny"; - -void -runtime·minit(void) -{ -} - -void -runtime·osinit(void) -{ -} - -void -runtime·goenvs(void) -{ - runtime·goenvs_unix(); -} - -void -runtime·initsig(int32 queue) -{ -} - -void -runtime·exit(int32) -{ - for(;;); -} - -// single processor, no interrupts, -// so no need for real concurrency or atomicity - -void -runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void)) -{ - USED(m, g, stk, fn); - runtime·throw("newosproc"); -} - -void -runtime·lock(Lock *l) -{ - if(m->locks < 0) - runtime·throw("lock count"); - m->locks++; - if(l->key != 0) - runtime·throw("deadlock"); - l->key = 1; -} - -void -runtime·unlock(Lock *l) -{ - m->locks--; - if(m->locks < 0) - runtime·throw("lock count"); - if(l->key != 1) - runtime·throw("unlock of unlocked lock"); - l->key = 0; -} - -void -runtime·destroylock(Lock *l) -{ - // nothing -} - -void -runtime·noteclear(Note *n) -{ - n->lock.key = 0; -} - -void -runtime·notewakeup(Note *n) -{ - n->lock.key = 1; -} - -void -runtime·notesleep(Note *n) -{ - if(n->lock.key != 1) - runtime·throw("notesleep"); -} - diff --git a/src/pkg/runtime/type.go b/src/pkg/runtime/type.go index d92fe5f2a..87268db4c 100644 --- a/src/pkg/runtime/type.go +++ b/src/pkg/runtime/type.go @@ -51,10 +51,8 @@ const ( kindUint32 kindUint64 kindUintptr - kindFloat kindFloat32 kindFloat64 - kindComplex kindComplex64 kindComplex128 kindArray @@ -72,7 +70,7 @@ const ( ) // Method on non-interface type -type method struct { +type _method struct { // underscore is to avoid collision with C name *string // name of method pkgPath *string // nil for exported Names; otherwise import path mtyp *Type // method type (without receiver) @@ -86,9 +84,9 @@ type method struct { // Using a pointer to this struct reduces the overall size required // to describe an unnamed type with no methods. type uncommonType struct { - name *string // name of type - pkgPath *string // import path; nil for built-in types like int, string - methods []method // methods associated with type + name *string // name of type + pkgPath *string // import path; nil for built-in types like int, string + methods []_method // methods associated with type } // BoolType represents a boolean type. @@ -153,7 +151,7 @@ type FuncType struct { } // Method on interface type -type imethod struct { +type _imethod struct { // underscore is to avoid collision with C name *string // name of method pkgPath *string // nil for exported Names; otherwise import path typ *Type // .(*FuncType) underneath @@ -162,7 +160,7 @@ type imethod struct { // InterfaceType represents an interface type. type InterfaceType struct { commonType - methods []imethod // sorted by hash + methods []_imethod // sorted by hash } // MapType represents a map type. diff --git a/src/pkg/runtime/type.h b/src/pkg/runtime/type.h index 4b5bd7ac2..c7d9dace2 100644 --- a/src/pkg/runtime/type.h +++ b/src/pkg/runtime/type.h @@ -4,6 +4,11 @@ /* * Runtime type representation; master is type.go + * + * The *Types here correspond 1-1 to type.go's *Type's, but are + * prefixed with an extra header of 2 pointers, corresponding to the + * interface{} structure, which itself is called type Type again on + * the Go side. */ typedef struct CommonType CommonType; @@ -41,10 +46,8 @@ enum { KindUint32, KindUint64, KindUintptr, - KindFloat, KindFloat32, KindFloat64, - KindComplex, KindComplex64, KindComplex128, KindArray, diff --git a/src/pkg/runtime/windows/386/defs.h b/src/pkg/runtime/windows/386/defs.h index f5a16367e..a2a882103 100644 --- a/src/pkg/runtime/windows/386/defs.h +++ b/src/pkg/runtime/windows/386/defs.h @@ -10,8 +10,69 @@ enum { PROT_EXEC = 0x4, MAP_ANON = 0x1, MAP_PRIVATE = 0x2, + EXCEPTION_ACCESS_VIOLATION = 0xc0000005, + EXCEPTION_BREAKPOINT = 0x80000003, + EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d, + EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e, + EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f, + EXCEPTION_FLT_OVERFLOW = 0xc0000091, + EXCEPTION_FLT_UNDERFLOW = 0xc0000093, + EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094, + EXCEPTION_INT_OVERFLOW = 0xc0000095, }; // Types #pragma pack on + +typedef struct ExceptionRecord ExceptionRecord; +struct ExceptionRecord { + uint32 ExceptionCode; + uint32 ExceptionFlags; + ExceptionRecord *ExceptionRecord; + void *ExceptionAddress; + uint32 NumberParameters; + uint32 ExceptionInformation[15]; +}; + +typedef struct FloatingSaveArea FloatingSaveArea; +struct FloatingSaveArea { + uint32 ControlWord; + uint32 StatusWord; + uint32 TagWord; + uint32 ErrorOffset; + uint32 ErrorSelector; + uint32 DataOffset; + uint32 DataSelector; + uint8 RegisterArea[80]; + uint32 Cr0NpxState; +}; + +typedef struct Context Context; +struct Context { + uint32 ContextFlags; + uint32 Dr0; + uint32 Dr1; + uint32 Dr2; + uint32 Dr3; + uint32 Dr6; + uint32 Dr7; + FloatingSaveArea FloatSave; + uint32 SegGs; + uint32 SegFs; + uint32 SegEs; + uint32 SegDs; + uint32 Edi; + uint32 Esi; + uint32 Ebx; + uint32 Edx; + uint32 Ecx; + uint32 Eax; + uint32 Ebp; + uint32 Eip; + uint32 SegCs; + uint32 EFlags; + uint32 Esp; + uint32 SegSs; + uint8 ExtendedRegisters[512]; +}; #pragma pack off diff --git a/src/pkg/runtime/windows/386/rt0.s b/src/pkg/runtime/windows/386/rt0.s index e379830fb..3b023de2f 100644 --- a/src/pkg/runtime/windows/386/rt0.s +++ b/src/pkg/runtime/windows/386/rt0.s @@ -3,4 +3,12 @@ // license that can be found in the LICENSE file. TEXT _rt0_386_windows(SB),7,$0 + // Set up SEH frame for bootstrap m + PUSHL $runtime·sigtramp(SB) + PUSHL 0(FS) + MOVL SP, 0(FS) + JMP _rt0_386(SB) + +DATA runtime·iswindows(SB)/4, $1 +GLOBL runtime·iswindows(SB), $4 diff --git a/src/pkg/runtime/windows/386/signal.c b/src/pkg/runtime/windows/386/signal.c index 2ae79e5b5..69178cdd0 100644 --- a/src/pkg/runtime/windows/386/signal.c +++ b/src/pkg/runtime/windows/386/signal.c @@ -3,6 +3,26 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "defs.h" +#include "os.h" + +void +runtime·dumpregs(Context *r) +{ + runtime·printf("eax %x\n", r->Eax); + runtime·printf("ebx %x\n", r->Ebx); + runtime·printf("ecx %x\n", r->Ecx); + runtime·printf("edx %x\n", r->Edx); + runtime·printf("edi %x\n", r->Edi); + runtime·printf("esi %x\n", r->Esi); + runtime·printf("ebp %x\n", r->Ebp); + runtime·printf("esp %x\n", r->Esp); + runtime·printf("eip %x\n", r->Eip); + runtime·printf("eflags %x\n", r->EFlags); + runtime·printf("cs %x\n", r->SegCs); + runtime·printf("fs %x\n", r->SegFs); + runtime·printf("gs %x\n", r->SegGs); +} void runtime·initsig(int32) @@ -15,3 +35,62 @@ runtime·signame(int32) return runtime·emptystring; } +uint32 +runtime·sighandler(ExceptionRecord *info, void *frame, Context *r) +{ + uintptr *sp; + G *gp; + + USED(frame); + + switch(info->ExceptionCode) { + case EXCEPTION_BREAKPOINT: + r->Eip--; // because 8l generates 2 bytes for INT3 + return 1; + } + + if((gp = m->curg) != nil && runtime·issigpanic(info->ExceptionCode)) { + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp->sig = info->ExceptionCode; + gp->sigcode0 = info->ExceptionInformation[0]; + gp->sigcode1 = info->ExceptionInformation[1]; + gp->sigpc = r->Eip; + + // Only push runtime·sigpanic if r->eip != 0. + // If r->eip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime·sigpanic instead. + // (Otherwise the trace will end at runtime·sigpanic and we + // won't get to see who faulted.) + if(r->Eip != 0) { + sp = (uintptr*)r->Esp; + *--sp = r->Eip; + r->Esp = (uintptr)sp; + } + r->Eip = (uintptr)runtime·sigpanic; + return 0; + } + + if(runtime·panicking) // traceback already printed + runtime·exit(2); + runtime·panicking = 1; + + runtime·printf("Exception %x %p %p\n", info->ExceptionCode, + info->ExceptionInformation[0], info->ExceptionInformation[1]); + + runtime·printf("PC=%x\n", r->Eip); + runtime·printf("\n"); + + if(runtime·gotraceback()){ + runtime·traceback((void*)r->Eip, (void*)r->Esp, 0, m->curg); + runtime·tracebackothers(m->curg); + runtime·dumpregs(r); + } + + runtime·breakpoint(); + runtime·exit(2); + return 0; +} diff --git a/src/pkg/runtime/windows/386/sys.s b/src/pkg/runtime/windows/386/sys.s index 7f99b34de..d1a8a49a9 100644 --- a/src/pkg/runtime/windows/386/sys.s +++ b/src/pkg/runtime/windows/386/sys.s @@ -5,7 +5,7 @@ #include "386/asm.h" // void *stdcall_raw(void *fn, int32 count, uintptr *args) -TEXT runtime·stdcall_raw(SB),7,$4 +TEXT runtime·stdcall_raw(SB),7,$0 // Copy arguments from stack. MOVL fn+0(FP), AX MOVL count+4(FP), CX // words @@ -14,17 +14,18 @@ TEXT runtime·stdcall_raw(SB),7,$4 // Switch to m->g0 if needed. get_tls(DI) MOVL m(DI), DX - MOVL g(DI), SI - MOVL SI, 0(SP) // save g - MOVL SP, m_gostack(DX) // save SP + MOVL 0(FS), SI + MOVL SI, m_sehframe(DX) MOVL m_g0(DX), SI CMPL g(DI), SI - JEQ 3(PC) + MOVL SP, BX + JEQ 2(PC) MOVL (m_sched+gobuf_sp)(DX), SP + PUSHL BX + PUSHL g(DI) MOVL SI, g(DI) // Copy args to new stack. - SUBL $(10*4), SP // padding MOVL CX, BX SALL $2, BX SUBL BX, SP // room for args @@ -38,28 +39,122 @@ TEXT runtime·stdcall_raw(SB),7,$4 // Restore original SP, g. get_tls(DI) - MOVL m(DI), DX - MOVL m_gostack(DX), SP // restore SP - MOVL 0(SP), SI // restore g - MOVL SI, g(DI) + POPL g(DI) + POPL SP // Someday the convention will be D is always cleared. CLD - RET + RET + +// faster get/set last error +TEXT runtime·getlasterror(SB),7,$0 + MOVL 0x34(FS), AX + RET + +TEXT runtime·setlasterror(SB),7,$0 + MOVL err+0(FP), AX + MOVL AX, 0x34(FS) + RET + +TEXT runtime·sigtramp(SB),7,$0 + PUSHL BP // cdecl + PUSHL 0(FS) + CALL runtime·sigtramp1(SB) + POPL 0(FS) + POPL BP + RET + +TEXT runtime·sigtramp1(SB),0,$16-28 + // unwinding? + MOVL info+12(FP), BX + MOVL 4(BX), CX // exception flags + ANDL $6, CX + MOVL $1, AX + JNZ sigdone + + // place ourselves at the top of the SEH chain to + // ensure SEH frames lie within thread stack bounds + MOVL frame+16(FP), CX // our SEH frame + MOVL CX, 0(FS) + + // copy arguments for call to sighandler + MOVL BX, 0(SP) + MOVL CX, 4(SP) + MOVL context+20(FP), BX + MOVL BX, 8(SP) + MOVL dispatcher+24(FP), BX + MOVL BX, 12(SP) + + CALL runtime·sighandler(SB) + TESTL AX, AX + JZ sigdone + + // call windows default handler early + MOVL 4(SP), BX // our SEH frame + MOVL 0(BX), BX // SEH frame of default handler + MOVL BX, 4(SP) // set establisher frame + CALL 4(BX) + +sigdone: + RET + +// Called from dynamic function created by ../thread.c compilecallback, +// running on Windows stack (not Go stack). +// BX, BP, SI, DI registers and DF flag are preserved +// as required by windows callback convention. +// AX = address of go func we need to call +// DX = total size of arguments +// +TEXT runtime·callbackasm+0(SB),7,$0 + LEAL 8(SP), CX + + // save registers as required for windows callback + PUSHL 0(FS) + PUSHL DI + PUSHL SI + PUSHL BP + PUSHL BX + PUSHL DX + PUSHL CX + PUSHL AX + + // reinstall our SEH handler + get_tls(CX) + MOVL m(CX), CX + MOVL m_sehframe(CX), CX + MOVL CX, 0(FS) + CLD + + CALL runtime·cgocallback(SB) + + // restore registers as required for windows callback + POPL CX + POPL CX + POPL DX + POPL BX + POPL BP + POPL SI + POPL DI + POPL 0(FS) + CLD + + RET // void tstart(M *newm); TEXT runtime·tstart(SB),7,$0 MOVL newm+4(SP), CX // m MOVL m_g0(CX), DX // g - MOVL SP, DI // remember stack + // Set up SEH frame + PUSHL $runtime·sigtramp(SB) + PUSHL 0(FS) + MOVL SP, 0(FS) // Layout new m scheduler stack on os stack. MOVL SP, AX - SUBL $256, AX // just some space for ourselves MOVL AX, g_stackbase(DX) - SUBL $8192, AX // stack size + SUBL $(64*1024), AX // stack size MOVL AX, g_stackguard(DX) // Set up tls. @@ -68,20 +163,17 @@ TEXT runtime·tstart(SB),7,$0 MOVL CX, m(SI) MOVL DX, g(SI) - // Use scheduler stack now. - MOVL g_stackbase(DX), SP - // Someday the convention will be D is always cleared. CLD - PUSHL DI // original stack - - CALL runtime·stackcheck(SB) // clobbers AX,CX + CALL runtime·stackcheck(SB) // clobbers AX,CX CALL runtime·mstart(SB) - POPL DI // original stack - MOVL DI, SP + // Pop SEH frame + MOVL 0(FS), SP + POPL 0(FS) + POPL CX RET @@ -107,12 +199,3 @@ TEXT runtime·setldt(SB),7,$0 MOVL address+4(FP), CX MOVL CX, 0x2c(FS) RET - -// for now, return 0,0. only used for internal performance monitoring. -TEXT runtime·gettime(SB),7,$0 - MOVL sec+0(FP), DI - MOVL $0, (DI) - MOVL $0, 4(DI) // zero extend 32 -> 64 bits - MOVL usec+4(FP), DI - MOVL $0, (DI) - RET diff --git a/src/pkg/runtime/windows/defs.c b/src/pkg/runtime/windows/defs.c index db5f1400e..5aac03c81 100644 --- a/src/pkg/runtime/windows/defs.c +++ b/src/pkg/runtime/windows/defs.c @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include <stdarg.h> +#include <windef.h> +#include <winbase.h> + enum { $PROT_NONE = 0, $PROT_READ = 1, @@ -10,4 +14,18 @@ enum { $MAP_ANON = 1, $MAP_PRIVATE = 2, + + $EXCEPTION_ACCESS_VIOLATION = STATUS_ACCESS_VIOLATION, + $EXCEPTION_BREAKPOINT = STATUS_BREAKPOINT, + $EXCEPTION_FLT_DENORMAL_OPERAND = STATUS_FLOAT_DENORMAL_OPERAND, + $EXCEPTION_FLT_DIVIDE_BY_ZERO = STATUS_FLOAT_DIVIDE_BY_ZERO, + $EXCEPTION_FLT_INEXACT_RESULT = STATUS_FLOAT_INEXACT_RESULT, + $EXCEPTION_FLT_OVERFLOW = STATUS_FLOAT_OVERFLOW, + $EXCEPTION_FLT_UNDERFLOW = STATUS_FLOAT_UNDERFLOW, + $EXCEPTION_INT_DIVIDE_BY_ZERO = STATUS_INTEGER_DIVIDE_BY_ZERO, + $EXCEPTION_INT_OVERFLOW = STATUS_INTEGER_OVERFLOW, }; + +typedef EXCEPTION_RECORD $ExceptionRecord; +typedef FLOATING_SAVE_AREA $FloatingSaveArea; +typedef CONTEXT $Context; diff --git a/src/pkg/runtime/windows/mem.c b/src/pkg/runtime/windows/mem.c index ba89887ea..19d11ce8d 100644 --- a/src/pkg/runtime/windows/mem.c +++ b/src/pkg/runtime/windows/mem.c @@ -15,16 +15,6 @@ enum { PAGE_EXECUTE_READWRITE = 0x40, }; -static void -abort(int8 *name) -{ - uintptr errno; - - errno = (uintptr)runtime·stdcall(runtime·GetLastError, 0); - runtime·printf("%s failed with errno=%d\n", name, errno); - runtime·throw(name); -} - #pragma dynimport runtime·VirtualAlloc VirtualAlloc "kernel32.dll" #pragma dynimport runtime·VirtualFree VirtualFree "kernel32.dll" extern void *runtime·VirtualAlloc; @@ -33,12 +23,8 @@ extern void *runtime·VirtualFree; void* runtime·SysAlloc(uintptr n) { - void *v; - - v = runtime·stdcall(runtime·VirtualAlloc, 4, nil, n, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); - if(v == 0) - abort("VirtualAlloc"); - return v; + mstats.sys += n; + return runtime·stdcall(runtime·VirtualAlloc, 4, nil, n, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); } void @@ -53,12 +39,25 @@ runtime·SysFree(void *v, uintptr n) { uintptr r; + mstats.sys -= n; r = (uintptr)runtime·stdcall(runtime·VirtualFree, 3, v, 0, MEM_RELEASE); if(r == 0) - abort("VirtualFree"); + runtime·throw("runtime: failed to release pages"); +} + +void* +runtime·SysReserve(void *v, uintptr n) +{ + return runtime·stdcall(runtime·VirtualAlloc, 4, v, n, MEM_RESERVE, 0); } void -runtime·SysMemInit(void) +runtime·SysMap(void *v, uintptr n) { + void *p; + + mstats.sys += n; + p = runtime·stdcall(runtime·VirtualAlloc, 4, v, n, MEM_COMMIT, PAGE_EXECUTE_READWRITE); + if(p != v) + runtime·throw("runtime: cannot map pages in arena address space"); } diff --git a/src/pkg/runtime/windows/os.h b/src/pkg/runtime/windows/os.h index 77d0d32a0..391eace5a 100644 --- a/src/pkg/runtime/windows/os.h +++ b/src/pkg/runtime/windows/os.h @@ -4,33 +4,23 @@ extern void *runtime·LoadLibraryEx; extern void *runtime·GetProcAddress; -extern void *runtime·GetLastError; - -// Get start address of symbol data in memory. -void *runtime·get_symdat_addr(void); // Call a Windows function with stdcall conventions, // and switch to os stack during the call. -void *runtime·stdcall_raw(void *fn, int32 count, uintptr *args); +void *runtime·stdcall_raw(void *fn, uintptr nargs, void *args); void *runtime·stdcall(void *fn, int32 count, ...); +uintptr runtime·syscall(void *fn, uintptr nargs, void *args, uintptr *err); + +uintptr runtime·getlasterror(void); +void runtime·setlasterror(uintptr err); -// Function to be called by windows CreateTread +// Function to be called by windows CreateThread // to start new os thread. uint32 runtime·tstart_stdcall(M *newm); -// Call stdcall Windows function StdcallParams.fn -// with params StdcallParams.args, -// followed immediately by GetLastError call. -// Both return values are returned in StdcallParams.r and -// StdcallParams.err. Will use os stack during the call. -typedef struct StdcallParams StdcallParams; -struct StdcallParams -{ - void *fn; - uintptr args[12]; - int32 n; - uintptr r; - uintptr err; -}; +uint32 runtime·issigpanic(uint32); +void runtime·sigpanic(void); -void runtime·syscall(StdcallParams *p); +// Windows dll function to go callback entry. +byte *runtime·compilecallback(Eface fn, bool cleanstack); +void *runtime·callbackasm(void); diff --git a/src/pkg/runtime/windows/runtime_defs.go b/src/pkg/runtime/windows/runtime_defs.go deleted file mode 100644 index 34a9b3259..000000000 --- a/src/pkg/runtime/windows/runtime_defs.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Go definitions of internal structures. Master is runtime.h - -package runtime - -import "unsafe" - -const ( - Windows = 1 -) - -// const ( Structrnd = sizeof(uintptr) ) - -type lock struct { - key uint32 - event unsafe.Pointer -} - -type note lock diff --git a/src/pkg/runtime/windows/syscall.goc b/src/pkg/runtime/windows/syscall.goc index d3057c540..85071e051 100644 --- a/src/pkg/runtime/windows/syscall.goc +++ b/src/pkg/runtime/windows/syscall.goc @@ -6,106 +6,62 @@ package syscall #include "runtime.h" #include "os.h" -func loadlibraryex(filename uintptr) (handle uint32) { - StdcallParams p; - p.fn = (void*)runtime·LoadLibraryEx; - p.args[0] = filename; - p.args[1] = 0; - p.args[2] = 0; - p.n = 3; - runtime·syscall(&p); - handle = p.r; +func loadlibraryex(filename uintptr) (handle uintptr) { + uintptr args[3] = { filename }; + handle = runtime·syscall(runtime·LoadLibraryEx, 3, args, nil); } -func getprocaddress(handle uint32, procname uintptr) (proc uintptr) { - StdcallParams p; - p.fn = (void*)runtime·GetProcAddress; - p.args[0] = handle; - p.args[1] = procname; - p.n = 2; - runtime·syscall(&p); - proc = p.r; +func getprocaddress(handle uintptr, procname uintptr) (proc uintptr) { + USED(procname); + proc = runtime·syscall(runtime·GetProcAddress, 2, &handle, nil); } -func Syscall(trap uintptr, a1 uintptr, a2 uintptr, a3 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { - StdcallParams p; - p.fn = (void*)trap; - p.args[0] = a1; - p.args[1] = a2; - p.args[2] = a3; - p.n = 3; - runtime·syscall(&p); - r1 = p.r; - r2 = 0; - err = p.err; +func NewCallback(fn Eface) (code uintptr) { + code = (uintptr)runtime·compilecallback(fn, true); } -func Syscall6(trap uintptr, a1 uintptr, a2 uintptr, a3 uintptr, a4 uintptr, a5 uintptr, a6 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { - StdcallParams p; - p.fn = (void*)trap; - p.args[0] = a1; - p.args[1] = a2; - p.args[2] = a3; - p.args[3] = a4; - p.args[4] = a5; - p.args[5] = a6; - p.n = 6; - runtime·syscall(&p); - r1 = p.r; +func Syscall(fn uintptr, nargs uintptr, a1 uintptr, a2 uintptr, a3 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { + USED(a2); + USED(a3); + r1 = runtime·syscall((void*)fn, nargs, &a1, &err); r2 = 0; - err = p.err; } -func Syscall9(trap uintptr, a1 uintptr, a2 uintptr, a3 uintptr, a4 uintptr, a5 uintptr, a6 uintptr, a7 uintptr, a8 uintptr, a9 uintptr) (r1 uintptr, r2 uintptr, lasterr uintptr) { - StdcallParams p; - p.fn = (void*)trap; - p.args[0] = a1; - p.args[1] = a2; - p.args[2] = a3; - p.args[3] = a4; - p.args[4] = a5; - p.args[5] = a6; - p.args[6] = a7; - p.args[7] = a8; - p.args[8] = a9; - p.n = 9; - runtime·syscall(&p); - r1 = p.r; +func Syscall6(fn uintptr, nargs uintptr, a1 uintptr, a2 uintptr, a3 uintptr, a4 uintptr, a5 uintptr, a6 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { + USED(a2); + USED(a3); + USED(a4); + USED(a5); + USED(a6); + r1 = runtime·syscall((void*)fn, nargs, &a1, &err); r2 = 0; - lasterr = p.err; } -func Syscall12(trap uintptr, a1 uintptr, a2 uintptr, a3 uintptr, a4 uintptr, a5 uintptr, a6 uintptr, a7 uintptr, a8 uintptr, a9 uintptr, a10 uintptr, a11 uintptr, a12 uintptr) (r1 uintptr, r2 uintptr, lasterr uintptr) { - StdcallParams p; - p.fn = (void*)trap; - p.args[0] = a1; - p.args[1] = a2; - p.args[2] = a3; - p.args[3] = a4; - p.args[4] = a5; - p.args[5] = a6; - p.args[6] = a7; - p.args[7] = a8; - p.args[8] = a9; - p.args[9] = a10; - p.args[10] = a11; - p.args[11] = a12; - p.n = 12; - runtime·syscall(&p); - r1 = p.r; +func Syscall9(fn uintptr, nargs uintptr, a1 uintptr, a2 uintptr, a3 uintptr, a4 uintptr, a5 uintptr, a6 uintptr, a7 uintptr, a8 uintptr, a9 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { + USED(a2); + USED(a3); + USED(a4); + USED(a5); + USED(a6); + USED(a7); + USED(a8); + USED(a9); + r1 = runtime·syscall((void*)fn, nargs, &a1, &err); r2 = 0; - lasterr = p.err; } -func RawSyscall(trap uintptr, a1 uintptr, a2 uintptr, a3 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { - StdcallParams p; - p.fn = (void*)trap; - p.args[0] = a1; - p.args[1] = a2; - p.args[2] = a3; - p.n = 3; - runtime·syscall(&p); - r1 = p.r; +func Syscall12(fn uintptr, nargs uintptr, a1 uintptr, a2 uintptr, a3 uintptr, a4 uintptr, a5 uintptr, a6 uintptr, a7 uintptr, a8 uintptr, a9 uintptr, a10 uintptr, a11 uintptr, a12 uintptr) (r1 uintptr, r2 uintptr, err uintptr) { + USED(a2); + USED(a3); + USED(a4); + USED(a5); + USED(a6); + USED(a7); + USED(a8); + USED(a9); + USED(a10); + USED(a11); + USED(a12); + r1 = runtime·syscall((void*)fn, nargs, &a1, &err); r2 = 0; - err = p.err; } diff --git a/src/pkg/runtime/windows/thread.c b/src/pkg/runtime/windows/thread.c index 9b5181337..278a5da69 100644 --- a/src/pkg/runtime/windows/thread.c +++ b/src/pkg/runtime/windows/thread.c @@ -3,48 +3,48 @@ // license that can be found in the LICENSE file. #include "runtime.h" +#include "type.h" +#include "defs.h" #include "os.h" -#pragma dynimport runtime·LoadLibraryEx LoadLibraryExA "kernel32.dll" -#pragma dynimport runtime·GetProcAddress GetProcAddress "kernel32.dll" #pragma dynimport runtime·CloseHandle CloseHandle "kernel32.dll" +#pragma dynimport runtime·CreateEvent CreateEventA "kernel32.dll" +#pragma dynimport runtime·CreateThread CreateThread "kernel32.dll" #pragma dynimport runtime·ExitProcess ExitProcess "kernel32.dll" +#pragma dynimport runtime·FreeEnvironmentStringsW FreeEnvironmentStringsW "kernel32.dll" +#pragma dynimport runtime·GetEnvironmentStringsW GetEnvironmentStringsW "kernel32.dll" +#pragma dynimport runtime·GetProcAddress GetProcAddress "kernel32.dll" #pragma dynimport runtime·GetStdHandle GetStdHandle "kernel32.dll" +#pragma dynimport runtime·LoadLibraryEx LoadLibraryExA "kernel32.dll" +#pragma dynimport runtime·QueryPerformanceCounter QueryPerformanceCounter "kernel32.dll" +#pragma dynimport runtime·QueryPerformanceFrequency QueryPerformanceFrequency "kernel32.dll" #pragma dynimport runtime·SetEvent SetEvent "kernel32.dll" +#pragma dynimport runtime·WaitForSingleObject WaitForSingleObject "kernel32.dll" #pragma dynimport runtime·WriteFile WriteFile "kernel32.dll" -#pragma dynimport runtime·GetLastError GetLastError "kernel32.dll" -#pragma dynimport runtime·SetLastError SetLastError "kernel32.dll" -// Also referenced by external packages extern void *runtime·CloseHandle; +extern void *runtime·CreateEvent; +extern void *runtime·CreateThread; extern void *runtime·ExitProcess; +extern void *runtime·FreeEnvironmentStringsW; +extern void *runtime·GetEnvironmentStringsW; +extern void *runtime·GetProcAddress; extern void *runtime·GetStdHandle; +extern void *runtime·LoadLibraryEx; +extern void *runtime·QueryPerformanceCounter; +extern void *runtime·QueryPerformanceFrequency; extern void *runtime·SetEvent; +extern void *runtime·WaitForSingleObject; extern void *runtime·WriteFile; -extern void *runtime·LoadLibraryEx; -extern void *runtime·GetProcAddress; -extern void *runtime·GetLastError; -extern void *runtime·SetLastError; - -#pragma dynimport runtime·CreateEvent CreateEventA "kernel32.dll" -#pragma dynimport runtime·CreateThread CreateThread "kernel32.dll" -#pragma dynimport runtime·WaitForSingleObject WaitForSingleObject "kernel32.dll" -extern void *runtime·CreateEvent; -extern void *runtime·CreateThread; -extern void *runtime·WaitForSingleObject; +static int64 timerfreq; void runtime·osinit(void) { + runtime·stdcall(runtime·QueryPerformanceFrequency, 1, &timerfreq); } -#pragma dynimport runtime·GetEnvironmentStringsW GetEnvironmentStringsW "kernel32.dll" -#pragma dynimport runtime·FreeEnvironmentStringsW FreeEnvironmentStringsW "kernel32.dll" - -extern void *runtime·GetEnvironmentStringsW; -extern void *runtime·FreeEnvironmentStringsW; - void runtime·goenvs(void) { @@ -193,6 +193,17 @@ runtime·minit(void) { } +void +runtime·gettime(int64 *sec, int32 *usec) +{ + int64 count; + + runtime·stdcall(runtime·QueryPerformanceCounter, 1, &count); + *sec = count / timerfreq; + count %= timerfreq; + *usec = count*1000000 / timerfreq; +} + // Calling stdcall on os stack. #pragma textflag 7 void * @@ -201,17 +212,124 @@ runtime·stdcall(void *fn, int32 count, ...) return runtime·stdcall_raw(fn, count, (uintptr*)(&count + 1)); } -void -runtime·syscall(StdcallParams *p) +uintptr +runtime·syscall(void *fn, uintptr nargs, void *args, uintptr *err) { - uintptr a; + G *oldlock; + uintptr ret; + + /* + * Lock g to m to ensure we stay on the same stack if we do a callback. + */ + oldlock = m->lockedg; + m->lockedg = g; + g->lockedm = m; runtime·entersyscall(); - // TODO(brainman): Move calls to SetLastError and GetLastError - // to stdcall_raw to speed up syscall. - a = 0; - runtime·stdcall_raw(runtime·SetLastError, 1, &a); - p->r = (uintptr)runtime·stdcall_raw((void*)p->fn, p->n, p->args); - p->err = (uintptr)runtime·stdcall_raw(runtime·GetLastError, 0, &a); + runtime·setlasterror(0); + ret = (uintptr)runtime·stdcall_raw(fn, nargs, args); + if(err) + *err = runtime·getlasterror(); runtime·exitsyscall(); + + m->lockedg = oldlock; + if(oldlock == nil) + g->lockedm = nil; + + return ret; +} + +uint32 +runtime·issigpanic(uint32 code) +{ + switch(code) { + case EXCEPTION_ACCESS_VIOLATION: + case EXCEPTION_INT_DIVIDE_BY_ZERO: + case EXCEPTION_INT_OVERFLOW: + case EXCEPTION_FLT_DENORMAL_OPERAND: + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + case EXCEPTION_FLT_INEXACT_RESULT: + case EXCEPTION_FLT_OVERFLOW: + case EXCEPTION_FLT_UNDERFLOW: + return 1; + } + return 0; +} + +void +runtime·sigpanic(void) +{ + switch(g->sig) { + case EXCEPTION_ACCESS_VIOLATION: + if(g->sigcode1 < 0x1000) + runtime·panicstring("invalid memory address or nil pointer dereference"); + runtime·printf("unexpected fault address %p\n", g->sigcode1); + runtime·throw("fault"); + case EXCEPTION_INT_DIVIDE_BY_ZERO: + runtime·panicstring("integer divide by zero"); + case EXCEPTION_INT_OVERFLOW: + runtime·panicstring("integer overflow"); + case EXCEPTION_FLT_DENORMAL_OPERAND: + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + case EXCEPTION_FLT_INEXACT_RESULT: + case EXCEPTION_FLT_OVERFLOW: + case EXCEPTION_FLT_UNDERFLOW: + runtime·panicstring("floating point error"); + } + runtime·throw("fault"); +} + +// Call back from windows dll into go. +byte * +runtime·compilecallback(Eface fn, bool cleanstack) +{ + Func *f; + int32 argsize, n; + byte *ret, *p; + + if(fn.type->kind != KindFunc) + runtime·panicstring("not a function"); + if((f = runtime·findfunc((uintptr)fn.data)) == nil) + runtime·throw("cannot find function"); + argsize = (f->args-2) * 4; + + // compute size of new fn. + // must match code laid out below. + n = 1+4; // MOVL fn, AX + n += 1+4; // MOVL argsize, DX + n += 1+4; // MOVL callbackasm, CX + n += 2; // CALL CX + n += 1; // RET + if(cleanstack) + n += 2; // ... argsize + + ret = p = runtime·mal(n); + + // MOVL fn, AX + *p++ = 0xb8; + *(uint32*)p = (uint32)fn.data; + p += 4; + + // MOVL argsize, DX + *p++ = 0xba; + *(uint32*)p = argsize; + p += 4; + + // MOVL callbackasm, CX + *p++ = 0xb9; + *(uint32*)p = (uint32)runtime·callbackasm; + p += 4; + + // CALL CX + *p++ = 0xff; + *p++ = 0xd1; + + // RET argsize? + if(cleanstack) { + *p++ = 0xc2; + *(uint16*)p = argsize; + } else + *p = 0xc3; + + return ret; } |