diff options
Diffstat (limited to 'src/pkg/runtime')
34 files changed, 599 insertions, 141 deletions
diff --git a/src/pkg/runtime/386/atomic.c b/src/pkg/runtime/386/atomic.c new file mode 100644 index 000000000..c031cc4f6 --- /dev/null +++ b/src/pkg/runtime/386/atomic.c @@ -0,0 +1,12 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" + +#pragma textflag 7 +uint32 +runtime·atomicload(uint32 volatile* addr) +{ + return *addr; +} diff --git a/src/pkg/runtime/Makefile b/src/pkg/runtime/Makefile index b122e0599..03f960cb8 100644 --- a/src/pkg/runtime/Makefile +++ b/src/pkg/runtime/Makefile @@ -12,9 +12,6 @@ SIZE_amd64=64 SIZE_arm=32 SIZE=$(SIZE_$(GOARCH)) -# TODO(kaib): fix register allocation to honor extern register so we -# can enable optimizations again. -CFLAGS_arm=-N CFLAGS_windows=-D__WINDOWS__ CFLAGS=-I$(GOOS) -I$(GOARCH) -I$(GOOS)/$(GOARCH) -wF $(CFLAGS_$(GOARCH)) $(CFLAGS_$(GOOS)) @@ -50,6 +47,7 @@ OFILES_arm=\ OFILES=\ asm.$O\ + atomic.$O\ cgocall.$O\ chan.$O\ closure.$O\ diff --git a/src/pkg/runtime/amd64/atomic.c b/src/pkg/runtime/amd64/atomic.c new file mode 100644 index 000000000..c031cc4f6 --- /dev/null +++ b/src/pkg/runtime/amd64/atomic.c @@ -0,0 +1,12 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" + +#pragma textflag 7 +uint32 +runtime·atomicload(uint32 volatile* addr) +{ + return *addr; +} diff --git a/src/pkg/runtime/arm/atomic.c b/src/pkg/runtime/arm/atomic.c new file mode 100644 index 000000000..9fd47bae7 --- /dev/null +++ b/src/pkg/runtime/arm/atomic.c @@ -0,0 +1,12 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" + +#pragma textflag 7 +uint32 +runtime·atomicload(uint32 volatile* addr) +{ + return runtime·xadd(addr, 0); +} diff --git a/src/pkg/runtime/cgo/Makefile b/src/pkg/runtime/cgo/Makefile index f26da2c51..7e752f127 100644 --- a/src/pkg/runtime/cgo/Makefile +++ b/src/pkg/runtime/cgo/Makefile @@ -10,6 +10,10 @@ ifeq ($(GOARCH),arm) ENABLED:=0 endif +ifeq ($(GOOS),plan9) +ENABLED:=0 +endif + ifeq ($(DISABLE_CGO),1) ENABLED:=0 endif diff --git a/src/pkg/runtime/cgo/darwin_386.c b/src/pkg/runtime/cgo/darwin_386.c index 13184f321..6d4e259be 100644 --- a/src/pkg/runtime/cgo/darwin_386.c +++ b/src/pkg/runtime/cgo/darwin_386.c @@ -113,11 +113,16 @@ libcgo_sys_thread_start(ThreadStart *ts) pthread_attr_t attr; pthread_t p; size_t size; + int err; pthread_attr_init(&attr); pthread_attr_getstacksize(&attr, &size); ts->g->stackguard = size; - pthread_create(&p, &attr, threadentry, ts); + err = pthread_create(&p, &attr, threadentry, ts); + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } } static void* diff --git a/src/pkg/runtime/cgo/darwin_amd64.c b/src/pkg/runtime/cgo/darwin_amd64.c index 38cd80a6f..3471044c0 100644 --- a/src/pkg/runtime/cgo/darwin_amd64.c +++ b/src/pkg/runtime/cgo/darwin_amd64.c @@ -83,11 +83,16 @@ libcgo_sys_thread_start(ThreadStart *ts) pthread_attr_t attr; pthread_t p; size_t size; + int err; pthread_attr_init(&attr); pthread_attr_getstacksize(&attr, &size); ts->g->stackguard = size; - pthread_create(&p, &attr, threadentry, ts); + err = pthread_create(&p, &attr, threadentry, ts); + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } } static void* diff --git a/src/pkg/runtime/cgo/freebsd_386.c b/src/pkg/runtime/cgo/freebsd_386.c index d08e1dee8..ae53201b4 100644 --- a/src/pkg/runtime/cgo/freebsd_386.c +++ b/src/pkg/runtime/cgo/freebsd_386.c @@ -20,11 +20,16 @@ libcgo_sys_thread_start(ThreadStart *ts) pthread_attr_t attr; pthread_t p; size_t size; + int err; pthread_attr_init(&attr); pthread_attr_getstacksize(&attr, &size); ts->g->stackguard = size; - pthread_create(&p, &attr, threadentry, ts); + err = pthread_create(&p, &attr, threadentry, ts); + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } } static void* diff --git a/src/pkg/runtime/cgo/freebsd_amd64.c b/src/pkg/runtime/cgo/freebsd_amd64.c index fe6ce391f..5afc1dfea 100644 --- a/src/pkg/runtime/cgo/freebsd_amd64.c +++ b/src/pkg/runtime/cgo/freebsd_amd64.c @@ -20,11 +20,16 @@ libcgo_sys_thread_start(ThreadStart *ts) pthread_attr_t attr; pthread_t p; size_t size; + int err; pthread_attr_init(&attr); pthread_attr_getstacksize(&attr, &size); ts->g->stackguard = size; - pthread_create(&p, &attr, threadentry, ts); + err = pthread_create(&p, &attr, threadentry, ts); + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } } static void* diff --git a/src/pkg/runtime/cgo/linux_386.c b/src/pkg/runtime/cgo/linux_386.c index 00322d4b7..e9df5ffdc 100644 --- a/src/pkg/runtime/cgo/linux_386.c +++ b/src/pkg/runtime/cgo/linux_386.c @@ -21,6 +21,7 @@ libcgo_sys_thread_start(ThreadStart *ts) pthread_attr_t attr; pthread_t p; size_t size; + int err; // Not sure why the memset is necessary here, // but without it, we get a bogus stack size @@ -30,7 +31,11 @@ libcgo_sys_thread_start(ThreadStart *ts) size = 0; pthread_attr_getstacksize(&attr, &size); ts->g->stackguard = size; - pthread_create(&p, &attr, threadentry, ts); + err = pthread_create(&p, &attr, threadentry, ts); + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } } static void* diff --git a/src/pkg/runtime/cgo/linux_amd64.c b/src/pkg/runtime/cgo/linux_amd64.c index e77c5ddfe..d9b8b3706 100644 --- a/src/pkg/runtime/cgo/linux_amd64.c +++ b/src/pkg/runtime/cgo/linux_amd64.c @@ -20,11 +20,16 @@ libcgo_sys_thread_start(ThreadStart *ts) pthread_attr_t attr; pthread_t p; size_t size; + int err; pthread_attr_init(&attr); pthread_attr_getstacksize(&attr, &size); ts->g->stackguard = size; - pthread_create(&p, &attr, threadentry, ts); + err = pthread_create(&p, &attr, threadentry, ts); + if (err != 0) { + fprintf(stderr, "runtime/cgo: pthread_create failed: %s\n", strerror(err)); + abort(); + } } static void* diff --git a/src/pkg/runtime/cgo/windows_amd64.c b/src/pkg/runtime/cgo/windows_amd64.c index dafe8cd9d..fd5b397ab 100755 --- a/src/pkg/runtime/cgo/windows_amd64.c +++ b/src/pkg/runtime/cgo/windows_amd64.c @@ -37,11 +37,21 @@ threadentry(void *v) ts.g->stackbase = (uintptr)&ts; /* - * libcgo_sys_thread_start set stackguard to stack size; - * change to actual guard pointer. - */ + * libcgo_sys_thread_start set stackguard to stack size; + * change to actual guard pointer. + */ ts.g->stackguard = (uintptr)&ts - ts.g->stackguard + 4096; - crosscall_386(ts.fn); + /* + * Set specific keys in thread local storage. + */ + asm volatile ( + "movq %%gs:0x58, %%rax\n" // MOVQ 0x58(GS), tmp + "movq %0, 0(%%rax)\n" // MOVQ g, 0(GS) + "movq %1, 8(%%rax)\n" // MOVQ m, 8(GS) + :: "r"(ts.g), "r"(ts.m) : "%rax" + ); + + crosscall_amd64(ts.fn); return nil; } diff --git a/src/pkg/runtime/debug/stack.go b/src/pkg/runtime/debug/stack.go index e5fae632b..a533a5c3b 100644 --- a/src/pkg/runtime/debug/stack.go +++ b/src/pkg/runtime/debug/stack.go @@ -52,7 +52,7 @@ func stack() []byte { if err != nil { continue } - lines = bytes.Split(data, []byte{'\n'}, -1) + lines = bytes.Split(data, []byte{'\n'}) lastFile = file } line-- // in stack trace, lines are 1-indexed but our array is 0-indexed diff --git a/src/pkg/runtime/debug/stack_test.go b/src/pkg/runtime/debug/stack_test.go index f4bdc4624..4aeea13ff 100644 --- a/src/pkg/runtime/debug/stack_test.go +++ b/src/pkg/runtime/debug/stack_test.go @@ -35,7 +35,7 @@ func (t T) method() []byte { */ func TestStack(t *testing.T) { b := T(0).method() - lines := strings.Split(string(b), "\n", -1) + lines := strings.Split(string(b), "\n") if len(lines) <= 6 { t.Fatal("too few lines") } diff --git a/src/pkg/runtime/error.go b/src/pkg/runtime/error.go index 289d78f49..6c37f888f 100644 --- a/src/pkg/runtime/error.go +++ b/src/pkg/runtime/error.go @@ -131,3 +131,8 @@ func printany(i interface{}) { print("(", typestring(i), ") ", i) } } + +// called from generated code +func panicwrap(pkg, typ, meth string) { + panic("value method " + pkg + "." + typ + "." + meth + " called using nil *" + typ + " pointer") +} diff --git a/src/pkg/runtime/linux/mem.c b/src/pkg/runtime/linux/mem.c index 38ca7e4a0..ad0fac6d3 100644 --- a/src/pkg/runtime/linux/mem.c +++ b/src/pkg/runtime/linux/mem.c @@ -91,6 +91,9 @@ runtime·SysMap(void *v, uintptr n) if(p != v && addrspace_free(v, n)) { // On some systems, mmap ignores v without // MAP_FIXED, so retry if the address space is free. + if(p > (void*)4096) { + runtime·munmap(p, n); + } p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); } if(p == (void*)ENOMEM) diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc index c55be9772..49ab24df8 100644 --- a/src/pkg/runtime/malloc.goc +++ b/src/pkg/runtime/malloc.goc @@ -229,7 +229,7 @@ runtime·allocmcache(void) return c; } -int32 runtime·sizeof_C_MStats = sizeof(MStats); +uintptr runtime·sizeof_C_MStats = sizeof(MStats); #define MaxArena32 (2U<<30) diff --git a/src/pkg/runtime/mem.go b/src/pkg/runtime/mem.go index fe505a329..c3316d44c 100644 --- a/src/pkg/runtime/mem.go +++ b/src/pkg/runtime/mem.go @@ -52,7 +52,7 @@ type MemStatsType struct { } } -var sizeof_C_MStats int // filled in by malloc.goc +var sizeof_C_MStats uintptr // filled in by malloc.goc func init() { if sizeof_C_MStats != unsafe.Sizeof(MemStats) { diff --git a/src/pkg/runtime/mkasmh.sh b/src/pkg/runtime/mkasmh.sh index 91d1bbe5d..328e2d5ba 100755 --- a/src/pkg/runtime/mkasmh.sh +++ b/src/pkg/runtime/mkasmh.sh @@ -14,7 +14,6 @@ EOF case "$GOARCH" in 386) # The offsets 0 and 4 are also known to: - # nacl/thread.c:/^newosproc # ../../cmd/8l/pass.c:/D_GS # ../../libcgo/linux_386.c:/^threadentry # ../../libcgo/darwin_386.c:/^threadentry @@ -62,14 +61,23 @@ case "$GOARCH" in esac ;; amd64) - # The offsets 0 and 8 are known to: - # ../../cmd/6l/pass.c:/D_GS - # ../../libcgo/linux_amd64.c:/^threadentry - # ../../libcgo/darwin_amd64.c:/^threadentry - # - echo '#define get_tls(r)' - echo '#define g(r) 0(GS)' - echo '#define m(r) 8(GS)' + case "$GOOS" in + windows) + echo '#define get_tls(r) MOVQ 0x58(GS), r' + echo '#define g(r) 0(r)' + echo '#define m(r) 8(r)' + ;; + *) + # The offsets 0 and 8 are known to: + # ../../cmd/6l/pass.c:/D_GS + # ../../libcgo/linux_amd64.c:/^threadentry + # ../../libcgo/darwin_amd64.c:/^threadentry + # + echo '#define get_tls(r)' + echo '#define g(r) 0(GS)' + echo '#define m(r) 8(GS)' + ;; + esac ;; arm) echo '#define g R10' diff --git a/src/pkg/runtime/plan9/386/sys.s b/src/pkg/runtime/plan9/386/sys.s index f760b782f..1cb570b68 100644 --- a/src/pkg/runtime/plan9/386/sys.s +++ b/src/pkg/runtime/plan9/386/sys.s @@ -9,6 +9,11 @@ TEXT runtime·setldt(SB),7,$0 RET +TEXT runtime·open(SB),7,$0 + MOVL $14, AX + INT $64 + RET + TEXT runtime·write(SB),7,$0 MOVL $20, AX INT $64 diff --git a/src/pkg/runtime/plan9/os.h b/src/pkg/runtime/plan9/os.h index 9444acc98..b2f7357ec 100644 --- a/src/pkg/runtime/plan9/os.h +++ b/src/pkg/runtime/plan9/os.h @@ -6,6 +6,14 @@ extern int32 runtime·write(int32 fd, void* buffer, int32 nbytes); extern void runtime·exits(int8* msg); extern int32 runtime·brk_(void*); +/* open */ +enum +{ + OREAD = 0, + OWRITE = 1, + ORDWR = 2 +}; + /* rfork */ enum { @@ -22,6 +30,28 @@ enum RFREND = (1<<13), RFNOMNT = (1<<14) }; + +typedef struct Tos Tos; +typedef intptr Plink; + +struct Tos { + struct /* Per process profiling */ + { + Plink *pp; /* known to be 0(ptr) */ + Plink *next; /* known to be 4(ptr) */ + Plink *last; + Plink *first; + uint32 pid; + uint32 what; + } prof; + uint64 cyclefreq; /* cycle clock frequency if there is one, 0 otherwise */ + int64 kcycles; /* cycles spent in kernel */ + int64 pcycles; /* cycles spent in process (kernel + user) */ + uint32 pid; /* might as well put the pid here */ + uint32 clock; + /* top of stack is here */ +}; + extern int32 runtime·rfork(int32 flags, void *stk, M *m, G *g, void (*fn)(void)); extern int32 runtime·plan9_semacquire(uint32 *addr, int32 block); extern int32 runtime·plan9_semrelease(uint32 *addr, int32 count); diff --git a/src/pkg/runtime/plan9/thread.c b/src/pkg/runtime/plan9/thread.c index 7c6ca45a3..ef9a23e8e 100644 --- a/src/pkg/runtime/plan9/thread.c +++ b/src/pkg/runtime/plan9/thread.c @@ -27,24 +27,48 @@ runtime·initsig(int32 queue) { } +extern Tos *_tos; void runtime·exit(int32) { + int32 fd; + uint8 buf[128]; + uint8 tmp[16]; + uint8 *p, *q; + int32 pid; + + runtime·memclr(buf, sizeof buf); + runtime·memclr(tmp, sizeof tmp); + pid = _tos->pid; + + /* build path string /proc/pid/notepg */ + for(q=tmp; pid > 0;) { + *q++ = '0' + (pid%10); + pid = pid/10; + } + p = buf; + runtime·mcpy((void*)p, (void*)"/proc/", 6); + p += 6; + for(q--; q >= tmp;) + *p++ = *q--; + runtime·mcpy((void*)p, (void*)"/notepg", 7); + + /* post interrupt note */ + fd = runtime·open(buf, OWRITE); + runtime·write(fd, "interrupt", 9); runtime·exits(nil); } void runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void)) { - USED(m, g, stk, fn); - m->tls[0] = m->id; // so 386 asm can find it if(0){ runtime·printf("newosproc stk=%p m=%p g=%p fn=%p rfork=%p id=%d/%d ostk=%p\n", stk, m, g, fn, runtime·rfork, m->id, m->tls[0], &m); } - if (runtime·rfork(RFPROC | RFMEM, stk, m, g, fn) < 0 ) + if (runtime·rfork(RFPROC|RFMEM|RFNOWAIT, stk, m, g, fn) < 0 ) runtime·throw("newosproc: rfork failed"); } diff --git a/src/pkg/runtime/pprof/pprof_test.go b/src/pkg/runtime/pprof/pprof_test.go index a060917a2..4486d5525 100644 --- a/src/pkg/runtime/pprof/pprof_test.go +++ b/src/pkg/runtime/pprof/pprof_test.go @@ -43,7 +43,7 @@ func TestCPUProfile(t *testing.T) { // Convert []byte to []uintptr. bytes := prof.Bytes() val := *(*[]uintptr)(unsafe.Pointer(&bytes)) - val = val[:len(bytes)/unsafe.Sizeof(uintptr(0))] + val = val[:len(bytes)/int(unsafe.Sizeof(uintptr(0)))] if len(val) < 10 { t.Fatalf("profile too short: %#x", val) diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index c5af8b754..a8f3a796a 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -77,7 +77,7 @@ struct Sched { }; Sched runtime·sched; -int32 gomaxprocs; +int32 runtime·gomaxprocs; // An m that is waiting for notewakeup(&m->havenextg). This may be // only be accessed while the scheduler lock is held. This is used to diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index f3ccff1bc..ad5da0a96 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -242,6 +242,11 @@ struct M uint32 fflag; // floating point compare flags #ifdef __WINDOWS__ void* sehframe; + +#ifdef _64BIT + void* gostack; +#endif + #endif }; @@ -416,7 +421,10 @@ int32 runtime·write(int32, void*, int32); int32 runtime·mincore(void*, uintptr, byte*); bool runtime·cas(uint32*, uint32, uint32); bool runtime·casp(void**, void*, void*); +// Don't confuse with XADD x86 instruction, +// this one is actually 'addx', that is, add-and-fetch. uint32 runtime·xadd(uint32 volatile*, int32); +uint32 runtime·atomicload(uint32 volatile*); void runtime·jmpdefer(byte*, void*); void runtime·exit1(int32); void runtime·ready(G*); diff --git a/src/pkg/runtime/sema.goc b/src/pkg/runtime/sema.goc index 1c77e87a5..ae84351ed 100644 --- a/src/pkg/runtime/sema.goc +++ b/src/pkg/runtime/sema.goc @@ -23,104 +23,68 @@ package runtime typedef struct Sema Sema; struct Sema { - uint32 *addr; + uint32 volatile *addr; G *g; Sema *prev; Sema *next; }; -// TODO: For now, a linked list; maybe a hash table of linked lists later. -static Sema *semfirst, *semlast; -static Lock semlock; +typedef struct SemaRoot SemaRoot; +struct SemaRoot +{ + Lock; + Sema *head; + Sema *tail; + // Number of waiters. Read w/o the lock. + uint32 volatile nwait; +}; + +// Prime to not correlate with any user patterns. +#define SEMTABLESZ 251 + +static union +{ + SemaRoot; + // Modern processors tend to have 64-byte cache lines, + // potentially with 128-byte effective cache line size for reading. + // While there are hypothetical architectures + // with 16-4096 byte cache lines, 128 looks like a good compromise. + uint8 pad[128]; +} semtable[SEMTABLESZ]; + +static SemaRoot* +semroot(uint32 *addr) +{ + return &semtable[((uintptr)addr >> 3) % SEMTABLESZ]; +} static void -semqueue(uint32 *addr, Sema *s) +semqueue(SemaRoot *root, uint32 volatile *addr, Sema *s) { + s->g = g; s->addr = addr; - s->g = nil; - - runtime·lock(&semlock); - s->prev = semlast; s->next = nil; - if(semlast) - semlast->next = s; + s->prev = root->tail; + if(root->tail) + root->tail->next = s; else - semfirst = s; - semlast = s; - runtime·unlock(&semlock); + root->head = s; + root->tail = s; } static void -semdequeue(Sema *s) +semdequeue(SemaRoot *root, Sema *s) { - runtime·lock(&semlock); if(s->next) s->next->prev = s->prev; else - semlast = s->prev; + root->tail = s->prev; if(s->prev) s->prev->next = s->next; else - semfirst = s->next; + root->head = s->next; s->prev = nil; s->next = nil; - runtime·unlock(&semlock); -} - -static void -semwakeup(uint32 *addr) -{ - Sema *s; - - runtime·lock(&semlock); - for(s=semfirst; s; s=s->next) { - if(s->addr == addr && s->g) { - runtime·ready(s->g); - s->g = nil; - break; - } - } - runtime·unlock(&semlock); -} - -// Step 1 of sleep: make ourselves available for wakeup. -// TODO(rsc): Maybe we can write a version without -// locks by using cas on s->g. Maybe not: I need to -// think more about whether it would be correct. -static void -semsleep1(Sema *s) -{ - runtime·lock(&semlock); - s->g = g; - runtime·unlock(&semlock); -} - -// Decided not to go through with it: undo step 1. -static void -semsleepundo1(Sema *s) -{ - runtime·lock(&semlock); - if(s->g != nil) { - s->g = nil; // back ourselves out - } else { - // If s->g == nil already, semwakeup - // already readied us. Since we never stopped - // running, readying us just set g->readyonstop. - // Clear it. - if(g->readyonstop == 0) - *(int32*)0x555 = 555; - g->readyonstop = 0; - } - runtime·unlock(&semlock); -} - -// Step 2: wait for the wakeup. -static void -semsleep2(Sema *s) -{ - USED(s); - g->status = Gwaiting; - runtime·gosched(); } static int32 @@ -128,52 +92,83 @@ cansemacquire(uint32 *addr) { uint32 v; - while((v = *addr) > 0) + while((v = runtime·atomicload(addr)) > 0) if(runtime·cas(addr, v, v-1)) return 1; return 0; } -// For now has no return value. -// Might return an ok (not interrupted) bool in the future? void -runtime·semacquire(uint32 *addr) +runtime·semacquire(uint32 volatile *addr) { Sema s; + SemaRoot *root; // Easy case. if(cansemacquire(addr)) return; // Harder case: - // queue - // try semacquire one more time, sleep if failed - // dequeue - // wake up one more guy to avoid races (TODO(rsc): maybe unnecessary?) - semqueue(addr, &s); + // increment waiter count + // try cansemacquire one more time, return if succeeded + // enqueue itself as a waiter + // sleep + // (waiter descriptor is dequeued by signaler) + root = semroot(addr); for(;;) { - semsleep1(&s); + runtime·lock(root); + // Add ourselves to nwait to disable "easy case" in semrelease. + runtime·xadd(&root->nwait, 1); + // Check cansemacquire to avoid missed wakeup. if(cansemacquire(addr)) { - semsleepundo1(&s); - break; + runtime·xadd(&root->nwait, -1); + runtime·unlock(root); + return; } - semsleep2(&s); + // Any semrelease after the cansemacquire knows we're waiting + // (we set nwait above), so go to sleep. + semqueue(root, addr, &s); + g->status = Gwaiting; + runtime·unlock(root); + runtime·gosched(); + if(cansemacquire(addr)) + return; } - semdequeue(&s); - semwakeup(addr); } void -runtime·semrelease(uint32 *addr) +runtime·semrelease(uint32 volatile *addr) { - uint32 v; + Sema *s; + SemaRoot *root; - for(;;) { - v = *addr; - if(runtime·cas(addr, v, v+1)) + root = semroot(addr); + runtime·xadd(addr, 1); + + // Easy case: no waiters? + // This check must happen after the xadd, to avoid a missed wakeup + // (see loop in semacquire). + if(runtime·atomicload(&root->nwait) == 0) + return; + + // Harder case: search for a waiter and wake it. + runtime·lock(root); + if(runtime·atomicload(&root->nwait) == 0) { + // The count is already consumed by another goroutine, + // so no need to wake up another goroutine. + runtime·unlock(root); + return; + } + for(s = root->head; s; s = s->next) { + if(s->addr == addr) { + runtime·xadd(&root->nwait, -1); + semdequeue(root, s); break; + } } - semwakeup(addr); + runtime·unlock(root); + if(s) + runtime·ready(s->g); } func Semacquire(addr *uint32) { diff --git a/src/pkg/runtime/sema_test.go b/src/pkg/runtime/sema_test.go new file mode 100644 index 000000000..d95bb1ec5 --- /dev/null +++ b/src/pkg/runtime/sema_test.go @@ -0,0 +1,100 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "runtime" + "sync/atomic" + "testing" +) + +func BenchmarkSemaUncontended(b *testing.B) { + type PaddedSem struct { + sem uint32 + pad [32]uint32 + } + const CallsPerSched = 1000 + procs := runtime.GOMAXPROCS(-1) + N := int32(b.N / CallsPerSched) + c := make(chan bool, procs) + for p := 0; p < procs; p++ { + go func() { + sem := new(PaddedSem) + for atomic.AddInt32(&N, -1) >= 0 { + runtime.Gosched() + for g := 0; g < CallsPerSched; g++ { + runtime.Semrelease(&sem.sem) + runtime.Semacquire(&sem.sem) + } + } + c <- true + }() + } + for p := 0; p < procs; p++ { + <-c + } +} + +func benchmarkSema(b *testing.B, block, work bool) { + const CallsPerSched = 1000 + const LocalWork = 100 + procs := runtime.GOMAXPROCS(-1) + N := int32(b.N / CallsPerSched) + c := make(chan bool, procs) + c2 := make(chan bool, procs/2) + sem := uint32(0) + if block { + for p := 0; p < procs/2; p++ { + go func() { + runtime.Semacquire(&sem) + c2 <- true + }() + } + } + for p := 0; p < procs; p++ { + go func() { + foo := 0 + for atomic.AddInt32(&N, -1) >= 0 { + runtime.Gosched() + for g := 0; g < CallsPerSched; g++ { + runtime.Semrelease(&sem) + if work { + for i := 0; i < LocalWork; i++ { + foo *= 2 + foo /= 2 + } + } + runtime.Semacquire(&sem) + } + } + c <- foo == 42 + runtime.Semrelease(&sem) + }() + } + if block { + for p := 0; p < procs/2; p++ { + <-c2 + } + } + for p := 0; p < procs; p++ { + <-c + } +} + +func BenchmarkSemaSyntNonblock(b *testing.B) { + benchmarkSema(b, false, false) +} + +func BenchmarkSemaSyntBlock(b *testing.B) { + benchmarkSema(b, true, false) +} + +func BenchmarkSemaWorkNonblock(b *testing.B) { + benchmarkSema(b, false, true) +} + +func BenchmarkSemaWorkBlock(b *testing.B) { + benchmarkSema(b, true, true) +} diff --git a/src/pkg/runtime/windows/amd64/defs.h b/src/pkg/runtime/windows/amd64/defs.h new file mode 100644 index 000000000..830c6a855 --- /dev/null +++ b/src/pkg/runtime/windows/amd64/defs.h @@ -0,0 +1,40 @@ +// g:\opensource\go\bin\godefs.exe -f -m64 defs.c + +// MACHINE GENERATED - DO NOT EDIT. + +// Constants +enum { + PROT_NONE = 0, + PROT_READ = 0x1, + PROT_WRITE = 0x2, + PROT_EXEC = 0x4, + MAP_ANON = 0x1, + MAP_PRIVATE = 0x2, + SIGINT = 0x2, + CTRL_C_EVENT = 0, + CTRL_BREAK_EVENT = 0x1, + EXCEPTION_ACCESS_VIOLATION = 0xc0000005, + EXCEPTION_BREAKPOINT = 0x80000003, + EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d, + EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e, + EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f, + EXCEPTION_FLT_OVERFLOW = 0xc0000091, + EXCEPTION_FLT_UNDERFLOW = 0xc0000093, + EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094, + EXCEPTION_INT_OVERFLOW = 0xc0000095, +}; + +// Types +#pragma pack on + +typedef struct ExceptionRecord ExceptionRecord; +struct ExceptionRecord { + uint32 ExceptionCode; + uint32 ExceptionFlags; + ExceptionRecord *ExceptionRecord; + void *ExceptionAddress; + uint32 NumberParameters; + byte pad_godefs_0[4]; + uint64 ExceptionInformation[15]; +}; +#pragma pack off diff --git a/src/pkg/runtime/windows/amd64/rt0.s b/src/pkg/runtime/windows/amd64/rt0.s new file mode 100644 index 000000000..e54e7edeb --- /dev/null +++ b/src/pkg/runtime/windows/amd64/rt0.s @@ -0,0 +1,10 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "amd64/asm.h" + +TEXT _rt0_amd64_windows(SB),7,$-8 + MOVQ $_rt0_amd64(SB), AX + MOVQ SP, DI + JMP AX diff --git a/src/pkg/runtime/windows/amd64/signal.c b/src/pkg/runtime/windows/amd64/signal.c new file mode 100644 index 000000000..1fc3eb060 --- /dev/null +++ b/src/pkg/runtime/windows/amd64/signal.c @@ -0,0 +1,20 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs.h" +#include "os.h" + +void +runtime·initsig(int32 queue) +{ +} + +void +runtime·resetcpuprofiler(int32 hz) +{ + // TODO: Enable profiling interrupts. + + m->profilehz = hz; +} diff --git a/src/pkg/runtime/windows/amd64/sys.s b/src/pkg/runtime/windows/amd64/sys.s new file mode 100644 index 000000000..b1eacfc82 --- /dev/null +++ b/src/pkg/runtime/windows/amd64/sys.s @@ -0,0 +1,129 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "amd64/asm.h" + +// void *stdcall_raw(void *fn, uintptr nargs, void *args) +TEXT runtime·stdcall_raw(SB),7,$8 + MOVQ fn+0(FP), AX + MOVQ nargs+8(FP), CX + MOVQ args+16(FP), R11 + + // Switch to m->g0 if needed. + get_tls(DI) + MOVQ m(DI), DX + MOVQ g(DI), SI + MOVQ SI, 0(SP) // save g + MOVQ SP, m_gostack(DX) // save SP + MOVQ m_g0(DX), SI + CMPQ g(DI), SI + JEQ 3(PC) + MOVQ (g_sched+gobuf_sp)(SI), SP + MOVQ SI, g(DI) + + SUBQ $0x60, SP + + // Copy args to new stack. + MOVQ SP, DI + MOVQ R11, SI + CLD + REP; MOVSQ + MOVQ 0(R11), CX + MOVQ 8(R11), DX + MOVQ 16(R11), R8 + MOVQ 24(R11), R9 + + // Call stdcall function. + CALL AX + + // Restore original SP, g. + get_tls(DI) + MOVQ m(DI), DX + MOVQ m_gostack(DX), SP // restore SP + MOVQ 0(SP), SI // restore g + MOVQ SI, g(DI) + + RET + +// faster get/set last error +TEXT runtime·getlasterror(SB),7,$0 + MOVQ 0x30(GS), AX + MOVL 0x68(AX), AX + RET + +TEXT runtime·setlasterror(SB),7,$0 + MOVL err+0(FP), AX + MOVQ 0x30(GS), CX + MOVL AX, 0x68(CX) + RET + +// Windows runs the ctrl handler in a new thread. +TEXT runtime·ctrlhandler(SB),7,$0 + // TODO + RET + +TEXT runtime·callbackasm(SB),7,$0 + // TODO + RET + +// void tstart(M *newm); +TEXT runtime·tstart(SB),7,$0 + MOVQ newm+8(SP), CX // m + MOVQ m_g0(CX), DX // g + + MOVQ SP, DI // remember stack + + // Layout new m scheduler stack on os stack. + MOVQ SP, AX + MOVQ AX, g_stackbase(DX) + SUBQ $(64*1024), AX // stack size + MOVQ AX, g_stackguard(DX) + + // Set up tls. + LEAQ m_tls(CX), SI + MOVQ SI, 0x58(GS) + MOVQ CX, m(SI) + MOVQ DX, g(SI) + + // Someday the convention will be D is always cleared. + CLD + + PUSHQ DI // original stack + + CALL runtime·stackcheck(SB) // clobbers AX,CX + + CALL runtime·mstart(SB) + + POPQ DI // original stack + MOVQ DI, SP + + RET + +// uint32 tstart_stdcall(M *newm); +TEXT runtime·tstart_stdcall(SB),7,$0 + MOVQ CX, BX // stdcall first arg in RCX + + PUSHQ BX + CALL runtime·tstart+0(SB) + POPQ BX + + // Adjust stack for stdcall to return properly. + MOVQ (SP), AX // save return address + ADDQ $8, SP // remove single parameter + MOVQ AX, (SP) // restore return address + + XORL AX, AX // return 0 == success + + RET + +TEXT runtime·notok(SB),7,$0 + MOVQ $0xf1, BP + MOVQ BP, (BP) + RET + +// set tls base to DI +TEXT runtime·settls(SB),7,$0 + MOVQ DI, 0x58(GS) + RET + diff --git a/src/pkg/runtime/windows/mem.c b/src/pkg/runtime/windows/mem.c index 54d77da37..5d2291fa3 100644 --- a/src/pkg/runtime/windows/mem.c +++ b/src/pkg/runtime/windows/mem.c @@ -24,7 +24,7 @@ void* runtime·SysAlloc(uintptr n) { mstats.sys += n; - return runtime·stdcall(runtime·VirtualAlloc, 4, nil, n, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); + return runtime·stdcall(runtime·VirtualAlloc, 4, nil, n, (uintptr)(MEM_COMMIT|MEM_RESERVE), (uintptr)PAGE_EXECUTE_READWRITE); } void @@ -40,7 +40,7 @@ runtime·SysFree(void *v, uintptr n) uintptr r; mstats.sys -= n; - r = (uintptr)runtime·stdcall(runtime·VirtualFree, 3, v, 0, MEM_RELEASE); + r = (uintptr)runtime·stdcall(runtime·VirtualFree, 3, v, (uintptr)0, (uintptr)MEM_RELEASE); if(r == 0) runtime·throw("runtime: failed to release pages"); } @@ -50,12 +50,12 @@ runtime·SysReserve(void *v, uintptr n) { // v is just a hint. // First try at v. - v = runtime·stdcall(runtime·VirtualAlloc, 4, v, n, MEM_RESERVE, PAGE_EXECUTE_READWRITE); + v = runtime·stdcall(runtime·VirtualAlloc, 4, v, n, (uintptr)MEM_RESERVE, (uintptr)PAGE_EXECUTE_READWRITE); if(v != nil) return v; // Next let the kernel choose the address. - return runtime·stdcall(runtime·VirtualAlloc, 4, nil, n, MEM_RESERVE, PAGE_EXECUTE_READWRITE); + return runtime·stdcall(runtime·VirtualAlloc, 4, nil, n, (uintptr)MEM_RESERVE, (uintptr)PAGE_EXECUTE_READWRITE); } void @@ -64,7 +64,7 @@ runtime·SysMap(void *v, uintptr n) void *p; mstats.sys += n; - p = runtime·stdcall(runtime·VirtualAlloc, 4, v, n, MEM_COMMIT, PAGE_EXECUTE_READWRITE); + p = runtime·stdcall(runtime·VirtualAlloc, 4, v, n, (uintptr)MEM_COMMIT, (uintptr)PAGE_EXECUTE_READWRITE); if(p != v) runtime·throw("runtime: cannot map pages in arena address space"); } diff --git a/src/pkg/runtime/windows/os.h b/src/pkg/runtime/windows/os.h index 77881e86e..bc9678733 100644 --- a/src/pkg/runtime/windows/os.h +++ b/src/pkg/runtime/windows/os.h @@ -7,6 +7,9 @@ extern void *runtime·GetProcAddress; // Call a Windows function with stdcall conventions, // and switch to os stack during the call. +#pragma varargck countpos runtime·stdcall 2 +#pragma varargck type runtime·stdcall void* +#pragma varargck type runtime·stdcall uintptr void *runtime·stdcall_raw(void *fn, uintptr nargs, void *args); void *runtime·stdcall(void *fn, int32 count, ...); uintptr runtime·syscall(void *fn, uintptr nargs, void *args, uintptr *err); diff --git a/src/pkg/runtime/windows/thread.c b/src/pkg/runtime/windows/thread.c index 81ad68033..5644fd5dd 100644 --- a/src/pkg/runtime/windows/thread.c +++ b/src/pkg/runtime/windows/thread.c @@ -45,7 +45,7 @@ void runtime·osinit(void) { runtime·stdcall(runtime·QueryPerformanceFrequency, 1, &timerfreq); - runtime·stdcall(runtime·SetConsoleCtrlHandler, 2, runtime·ctrlhandler, 1); + runtime·stdcall(runtime·SetConsoleCtrlHandler, 2, runtime·ctrlhandler, (uintptr)1); } void @@ -81,7 +81,7 @@ runtime·goenvs(void) void runtime·exit(int32 code) { - runtime·stdcall(runtime·ExitProcess, 1, code); + runtime·stdcall(runtime·ExitProcess, 1, (uintptr)code); } int32 @@ -93,15 +93,15 @@ runtime·write(int32 fd, void *buf, int32 n) written = 0; switch(fd) { case 1: - handle = runtime·stdcall(runtime·GetStdHandle, 1, -11); + handle = runtime·stdcall(runtime·GetStdHandle, 1, (uintptr)-11); break; case 2: - handle = runtime·stdcall(runtime·GetStdHandle, 1, -12); + handle = runtime·stdcall(runtime·GetStdHandle, 1, (uintptr)-12); break; default: return -1; } - runtime·stdcall(runtime·WriteFile, 5, handle, buf, n, &written, 0); + runtime·stdcall(runtime·WriteFile, 5, handle, buf, (uintptr)n, &written, (uintptr)0); return written; } @@ -111,7 +111,7 @@ initevent(void **pevent) { void *event; - event = runtime·stdcall(runtime·CreateEvent, 4, 0, 0, 0, 0); + event = runtime·stdcall(runtime·CreateEvent, 4, (uintptr)0, (uintptr)0, (uintptr)0, (uintptr)0); if(!runtime·casp(pevent, 0, event)) { // Someone else filled it in. Use theirs. runtime·stdcall(runtime·CloseHandle, 1, event); @@ -126,7 +126,7 @@ eventlock(Lock *l) initevent(&l->event); if(runtime·xadd(&l->key, 1) > 1) // someone else has it; wait - runtime·stdcall(runtime·WaitForSingleObject, 2, l->event, -1); + runtime·stdcall(runtime·WaitForSingleObject, 2, l->event, (uintptr)-1); } static void @@ -190,7 +190,7 @@ runtime·newosproc(M *m, G *g, void *stk, void (*fn)(void)) USED(g); // assuming g = m->g0 USED(fn); // assuming fn = mstart - thandle = runtime·stdcall(runtime·CreateThread, 6, 0, 0, runtime·tstart_stdcall, m, 0, 0); + thandle = runtime·stdcall(runtime·CreateThread, 6, (uintptr)0, (uintptr)0, runtime·tstart_stdcall, m, (uintptr)0, (uintptr)0); if(thandle == 0) { runtime·printf("runtime: failed to create new OS thread (have %d already; errno=%d)\n", runtime·mcount(), runtime·getlasterror()); runtime·throw("runtime.newosproc"); @@ -219,7 +219,7 @@ runtime·gettime(int64 *sec, int32 *usec) void * runtime·stdcall(void *fn, int32 count, ...) { - return runtime·stdcall_raw(fn, count, (uintptr*)(&count + 1)); + return runtime·stdcall_raw(fn, count, (uintptr*)&count + 1); } uintptr |