diff options
Diffstat (limited to 'src/pkg/runtime')
157 files changed, 5814 insertions, 3595 deletions
diff --git a/src/pkg/runtime/alg.c b/src/pkg/runtime/alg.c index ad85b43ae..2dc821256 100644 --- a/src/pkg/runtime/alg.c +++ b/src/pkg/runtime/alg.c @@ -8,6 +8,8 @@ #define M0 (sizeof(uintptr)==4 ? 2860486313UL : 33054211828000289ULL) #define M1 (sizeof(uintptr)==4 ? 3267000013UL : 23344194077549503ULL) +static bool use_aeshash; + /* * map and chan helpers for * dealing with unknown types @@ -17,6 +19,10 @@ runtime·memhash(uintptr *h, uintptr s, void *a) { byte *b; uintptr hash; + if(use_aeshash) { + runtime·aeshash(h, s, a); + return; + } b = a; hash = M0 ^ *h; @@ -467,6 +473,42 @@ runtime·algarray[] = // Runtime helpers. +// used in asm_{386,amd64}.s +byte runtime·aeskeysched[HashRandomBytes]; + +void +runtime·hashinit(void) +{ + // Install aes hash algorithm if we have the instructions we need + if((runtime·cpuid_ecx & (1 << 25)) != 0 && // aes (aesenc) + (runtime·cpuid_ecx & (1 << 9)) != 0 && // sse3 (pshufb) + (runtime·cpuid_ecx & (1 << 19)) != 0) { // sse4.1 (pinsr{d,q}) + byte *rnd; + int32 n; + use_aeshash = true; + runtime·algarray[AMEM].hash = runtime·aeshash; + runtime·algarray[AMEM8].hash = runtime·aeshash; + runtime·algarray[AMEM16].hash = runtime·aeshash; + runtime·algarray[AMEM32].hash = runtime·aeshash32; + runtime·algarray[AMEM64].hash = runtime·aeshash64; + runtime·algarray[AMEM128].hash = runtime·aeshash; + runtime·algarray[ASTRING].hash = runtime·aeshashstr; + + // Initialize with random data so hash collisions will be hard to engineer. + runtime·get_random_data(&rnd, &n); + if(n > HashRandomBytes) + n = HashRandomBytes; + runtime·memmove(runtime·aeskeysched, rnd, n); + if(n < HashRandomBytes) { + // Not very random, but better than nothing. + int64 t = runtime·nanotime(); + while (n < HashRandomBytes) { + runtime·aeskeysched[n++] = (int8)(t >> (8 * (n % 8))); + } + } + } +} + // func equal(t *Type, x T, y T) (ret bool) #pragma textflag 7 void diff --git a/src/pkg/runtime/arch_386.h b/src/pkg/runtime/arch_386.h index 4df795f71..62ed11b40 100644 --- a/src/pkg/runtime/arch_386.h +++ b/src/pkg/runtime/arch_386.h @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + enum { thechar = '8', BigEndian = 0, diff --git a/src/pkg/runtime/arch_amd64.h b/src/pkg/runtime/arch_amd64.h index e83dc9105..a5e43ca8d 100644 --- a/src/pkg/runtime/arch_amd64.h +++ b/src/pkg/runtime/arch_amd64.h @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + enum { thechar = '6', BigEndian = 0, diff --git a/src/pkg/runtime/arch_arm.h b/src/pkg/runtime/arch_arm.h index f6af58514..bb65d3faf 100644 --- a/src/pkg/runtime/arch_arm.h +++ b/src/pkg/runtime/arch_arm.h @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + enum { thechar = '5', BigEndian = 0, diff --git a/src/pkg/runtime/asm_386.s b/src/pkg/runtime/asm_386.s index 96f04e0ae..6bcacf4cc 100644 --- a/src/pkg/runtime/asm_386.s +++ b/src/pkg/runtime/asm_386.s @@ -6,8 +6,8 @@ TEXT _rt0_386(SB),7,$0 // copy arguments forward on an even stack - MOVL 0(SP), AX // argc - LEAL 4(SP), BX // argv + MOVL argc+0(FP), AX + MOVL argv+4(FP), BX SUBL $128, SP // plenty of scratch ANDL $~15, SP MOVL AX, 120(SP) // save argc, argv away @@ -20,15 +20,25 @@ TEXT _rt0_386(SB),7,$0 MOVL BX, g_stackguard(BP) MOVL SP, g_stackbase(BP) + // find out information about the processor we're on + MOVL $0, AX + CPUID + CMPL AX, $0 + JE nocpuinfo + MOVL $1, AX + CPUID + MOVL CX, runtime·cpuid_ecx(SB) + MOVL DX, runtime·cpuid_edx(SB) +nocpuinfo: + // if there is an _cgo_init, call it to let it // initialize and to set up GS. if not, // we set up GS ourselves. MOVL _cgo_init(SB), AX TESTL AX, AX JZ needtls - PUSHL BP + MOVL BP, 0(SP) CALL AX - POPL BP // skip runtime·ldt0setup(SB) and tls test after _cgo_init for non-windows CMPL runtime·iswindows(SB), $0 JEQ ok @@ -72,6 +82,7 @@ ok: MOVL AX, 4(SP) CALL runtime·args(SB) CALL runtime·osinit(SB) + CALL runtime·hashinit(SB) CALL runtime·schedinit(SB) // create a new goroutine to start program @@ -706,7 +717,261 @@ TEXT runtime·stackguard(SB),7,$0 get_tls(CX) MOVL g(CX), BX MOVL g_stackguard(BX), DX - MOVL DX, guard+4(FP) + MOVL DX, limit+4(FP) RET GLOBL runtime·tls0(SB), $32 + +// hash function using AES hardware instructions +TEXT runtime·aeshash(SB),7,$0 + MOVL 4(SP), DX // ptr to hash value + MOVL 8(SP), CX // size + MOVL 12(SP), AX // ptr to data + JMP runtime·aeshashbody(SB) + +TEXT runtime·aeshashstr(SB),7,$0 + MOVL 4(SP), DX // ptr to hash value + MOVL 12(SP), AX // ptr to string struct + MOVL 4(AX), CX // length of string + MOVL (AX), AX // string data + JMP runtime·aeshashbody(SB) + +// AX: data +// CX: length +// DX: ptr to seed input / hash output +TEXT runtime·aeshashbody(SB),7,$0 + MOVL (DX), X0 // seed to low 32 bits of xmm0 + PINSRD $1, CX, X0 // size to next 32 bits of xmm0 + MOVO runtime·aeskeysched+0(SB), X2 + MOVO runtime·aeskeysched+16(SB), X3 +aesloop: + CMPL CX, $16 + JB aesloopend + MOVOU (AX), X1 + AESENC X2, X0 + AESENC X1, X0 + SUBL $16, CX + ADDL $16, AX + JMP aesloop +aesloopend: + TESTL CX, CX + JE finalize // no partial block + + TESTL $16, AX + JNE highpartial + + // address ends in 0xxxx. 16 bytes loaded + // at this address won't cross a page boundary, so + // we can load it directly. + MOVOU (AX), X1 + ADDL CX, CX + PAND masks(SB)(CX*8), X1 + JMP partial +highpartial: + // address ends in 1xxxx. Might be up against + // a page boundary, so load ending at last byte. + // Then shift bytes down using pshufb. + MOVOU -16(AX)(CX*1), X1 + ADDL CX, CX + PSHUFB shifts(SB)(CX*8), X1 +partial: + // incorporate partial block into hash + AESENC X3, X0 + AESENC X1, X0 +finalize: + // finalize hash + AESENC X2, X0 + AESENC X3, X0 + AESENC X2, X0 + MOVL X0, (DX) + RET + +TEXT runtime·aeshash32(SB),7,$0 + MOVL 4(SP), DX // ptr to hash value + MOVL 12(SP), AX // ptr to data + MOVL (DX), X0 // seed + PINSRD $1, (AX), X0 // data + AESENC runtime·aeskeysched+0(SB), X0 + AESENC runtime·aeskeysched+16(SB), X0 + AESENC runtime·aeskeysched+0(SB), X0 + MOVL X0, (DX) + RET + +TEXT runtime·aeshash64(SB),7,$0 + MOVL 4(SP), DX // ptr to hash value + MOVL 12(SP), AX // ptr to data + MOVQ (AX), X0 // data + PINSRD $2, (DX), X0 // seed + AESENC runtime·aeskeysched+0(SB), X0 + AESENC runtime·aeskeysched+16(SB), X0 + AESENC runtime·aeskeysched+0(SB), X0 + MOVL X0, (DX) + RET + + +// simple mask to get rid of data in the high part of the register. +TEXT masks(SB),7,$0 + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + + LONG $0x000000ff + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + + LONG $0x0000ffff + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + + LONG $0x00ffffff + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffffff + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffffff + LONG $0x000000ff + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffffff + LONG $0x0000ffff + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffffff + LONG $0x00ffffff + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffffff + LONG $0xffffffff + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffffff + LONG $0xffffffff + LONG $0x000000ff + LONG $0x00000000 + + LONG $0xffffffff + LONG $0xffffffff + LONG $0x0000ffff + LONG $0x00000000 + + LONG $0xffffffff + LONG $0xffffffff + LONG $0x00ffffff + LONG $0x00000000 + + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + LONG $0x00000000 + + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + LONG $0x000000ff + + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + LONG $0x0000ffff + + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + LONG $0x00ffffff + + // these are arguments to pshufb. They move data down from + // the high bytes of the register to the low bytes of the register. + // index is how many bytes to move. +TEXT shifts(SB),7,$0 + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + LONG $0x00000000 + + LONG $0xffffff0f + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + + LONG $0xffff0f0e + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + + LONG $0xff0f0e0d + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + + LONG $0x0f0e0d0c + LONG $0xffffffff + LONG $0xffffffff + LONG $0xffffffff + + LONG $0x0e0d0c0b + LONG $0xffffff0f + LONG $0xffffffff + LONG $0xffffffff + + LONG $0x0d0c0b0a + LONG $0xffff0f0e + LONG $0xffffffff + LONG $0xffffffff + + LONG $0x0c0b0a09 + LONG $0xff0f0e0d + LONG $0xffffffff + LONG $0xffffffff + + LONG $0x0b0a0908 + LONG $0x0f0e0d0c + LONG $0xffffffff + LONG $0xffffffff + + LONG $0x0a090807 + LONG $0x0e0d0c0b + LONG $0xffffff0f + LONG $0xffffffff + + LONG $0x09080706 + LONG $0x0d0c0b0a + LONG $0xffff0f0e + LONG $0xffffffff + + LONG $0x08070605 + LONG $0x0c0b0a09 + LONG $0xff0f0e0d + LONG $0xffffffff + + LONG $0x07060504 + LONG $0x0b0a0908 + LONG $0x0f0e0d0c + LONG $0xffffffff + + LONG $0x06050403 + LONG $0x0a090807 + LONG $0x0e0d0c0b + LONG $0xffffff0f + + LONG $0x05040302 + LONG $0x09080706 + LONG $0x0d0c0b0a + LONG $0xffff0f0e + + LONG $0x04030201 + LONG $0x08070605 + LONG $0x0c0b0a09 + LONG $0xff0f0e0d + diff --git a/src/pkg/runtime/asm_amd64.s b/src/pkg/runtime/asm_amd64.s index 987958498..f4cfa576e 100644 --- a/src/pkg/runtime/asm_amd64.s +++ b/src/pkg/runtime/asm_amd64.s @@ -6,8 +6,8 @@ TEXT _rt0_amd64(SB),7,$-8 // copy arguments forward on an even stack - MOVQ 0(DI), AX // argc - LEAQ 8(DI), BX // argv + MOVQ DI, AX // argc + MOVQ SI, BX // argv SUBQ $(4*8+7), SP // 2args 2auto ANDQ $~15, SP MOVQ AX, 16(SP) @@ -20,6 +20,17 @@ TEXT _rt0_amd64(SB),7,$-8 MOVQ BX, g_stackguard(DI) MOVQ SP, g_stackbase(DI) + // find out information about the processor we're on + MOVQ $0, AX + CPUID + CMPQ AX, $0 + JE nocpuinfo + MOVQ $1, AX + CPUID + MOVL CX, runtime·cpuid_ecx(SB) + MOVL DX, runtime·cpuid_edx(SB) +nocpuinfo: + // if there is an _cgo_init, call it. MOVQ _cgo_init(SB), AX TESTQ AX, AX @@ -65,6 +76,7 @@ ok: MOVQ AX, 8(SP) CALL runtime·args(SB) CALL runtime·osinit(SB) + CALL runtime·hashinit(SB) CALL runtime·schedinit(SB) // create a new goroutine to start program @@ -442,6 +454,12 @@ TEXT runtime·xchg(SB), 7, $0 XCHGL AX, 0(BX) RET +TEXT runtime·xchg64(SB), 7, $0 + MOVQ 8(SP), BX + MOVQ 16(SP), AX + XCHGQ AX, 0(BX) + RET + TEXT runtime·procyield(SB),7,$0 MOVL 8(SP), AX again: @@ -719,7 +737,165 @@ TEXT runtime·stackguard(SB),7,$0 get_tls(CX) MOVQ g(CX), BX MOVQ g_stackguard(BX), DX - MOVQ DX, guard+8(FP) + MOVQ DX, limit+8(FP) RET GLOBL runtime·tls0(SB), $64 + +// hash function using AES hardware instructions +TEXT runtime·aeshash(SB),7,$0 + MOVQ 8(SP), DX // ptr to hash value + MOVQ 16(SP), CX // size + MOVQ 24(SP), AX // ptr to data + JMP runtime·aeshashbody(SB) + +TEXT runtime·aeshashstr(SB),7,$0 + MOVQ 8(SP), DX // ptr to hash value + MOVQ 24(SP), AX // ptr to string struct + MOVQ 8(AX), CX // length of string + MOVQ (AX), AX // string data + JMP runtime·aeshashbody(SB) + +// AX: data +// CX: length +// DX: ptr to seed input / hash output +TEXT runtime·aeshashbody(SB),7,$0 + MOVQ (DX), X0 // seed to low 64 bits of xmm0 + PINSRQ $1, CX, X0 // size to high 64 bits of xmm0 + MOVO runtime·aeskeysched+0(SB), X2 + MOVO runtime·aeskeysched+16(SB), X3 +aesloop: + CMPQ CX, $16 + JB aesloopend + MOVOU (AX), X1 + AESENC X2, X0 + AESENC X1, X0 + SUBQ $16, CX + ADDQ $16, AX + JMP aesloop +aesloopend: + TESTQ CX, CX + JE finalize // no partial block + + TESTQ $16, AX + JNE highpartial + + // address ends in 0xxxx. 16 bytes loaded + // at this address won't cross a page boundary, so + // we can load it directly. + MOVOU (AX), X1 + ADDQ CX, CX + PAND masks(SB)(CX*8), X1 + JMP partial +highpartial: + // address ends in 1xxxx. Might be up against + // a page boundary, so load ending at last byte. + // Then shift bytes down using pshufb. + MOVOU -16(AX)(CX*1), X1 + ADDQ CX, CX + PSHUFB shifts(SB)(CX*8), X1 +partial: + // incorporate partial block into hash + AESENC X3, X0 + AESENC X1, X0 +finalize: + // finalize hash + AESENC X2, X0 + AESENC X3, X0 + AESENC X2, X0 + MOVQ X0, (DX) + RET + +TEXT runtime·aeshash32(SB),7,$0 + MOVQ 8(SP), DX // ptr to hash value + MOVQ 24(SP), AX // ptr to data + MOVQ (DX), X0 // seed + PINSRD $2, (AX), X0 // data + AESENC runtime·aeskeysched+0(SB), X0 + AESENC runtime·aeskeysched+16(SB), X0 + AESENC runtime·aeskeysched+0(SB), X0 + MOVQ X0, (DX) + RET + +TEXT runtime·aeshash64(SB),7,$0 + MOVQ 8(SP), DX // ptr to hash value + MOVQ 24(SP), AX // ptr to data + MOVQ (DX), X0 // seed + PINSRQ $1, (AX), X0 // data + AESENC runtime·aeskeysched+0(SB), X0 + AESENC runtime·aeskeysched+16(SB), X0 + AESENC runtime·aeskeysched+0(SB), X0 + MOVQ X0, (DX) + RET + +// simple mask to get rid of data in the high part of the register. +TEXT masks(SB),7,$0 + QUAD $0x0000000000000000 + QUAD $0x0000000000000000 + QUAD $0x00000000000000ff + QUAD $0x0000000000000000 + QUAD $0x000000000000ffff + QUAD $0x0000000000000000 + QUAD $0x0000000000ffffff + QUAD $0x0000000000000000 + QUAD $0x00000000ffffffff + QUAD $0x0000000000000000 + QUAD $0x000000ffffffffff + QUAD $0x0000000000000000 + QUAD $0x0000ffffffffffff + QUAD $0x0000000000000000 + QUAD $0x00ffffffffffffff + QUAD $0x0000000000000000 + QUAD $0xffffffffffffffff + QUAD $0x0000000000000000 + QUAD $0xffffffffffffffff + QUAD $0x00000000000000ff + QUAD $0xffffffffffffffff + QUAD $0x000000000000ffff + QUAD $0xffffffffffffffff + QUAD $0x0000000000ffffff + QUAD $0xffffffffffffffff + QUAD $0x00000000ffffffff + QUAD $0xffffffffffffffff + QUAD $0x000000ffffffffff + QUAD $0xffffffffffffffff + QUAD $0x0000ffffffffffff + QUAD $0xffffffffffffffff + QUAD $0x00ffffffffffffff + + // these are arguments to pshufb. They move data down from + // the high bytes of the register to the low bytes of the register. + // index is how many bytes to move. +TEXT shifts(SB),7,$0 + QUAD $0x0000000000000000 + QUAD $0x0000000000000000 + QUAD $0xffffffffffffff0f + QUAD $0xffffffffffffffff + QUAD $0xffffffffffff0f0e + QUAD $0xffffffffffffffff + QUAD $0xffffffffff0f0e0d + QUAD $0xffffffffffffffff + QUAD $0xffffffff0f0e0d0c + QUAD $0xffffffffffffffff + QUAD $0xffffff0f0e0d0c0b + QUAD $0xffffffffffffffff + QUAD $0xffff0f0e0d0c0b0a + QUAD $0xffffffffffffffff + QUAD $0xff0f0e0d0c0b0a09 + QUAD $0xffffffffffffffff + QUAD $0x0f0e0d0c0b0a0908 + QUAD $0xffffffffffffffff + QUAD $0x0e0d0c0b0a090807 + QUAD $0xffffffffffffff0f + QUAD $0x0d0c0b0a09080706 + QUAD $0xffffffffffff0f0e + QUAD $0x0c0b0a0908070605 + QUAD $0xffffffffff0f0e0d + QUAD $0x0b0a090807060504 + QUAD $0xffffffff0f0e0d0c + QUAD $0x0a09080706050403 + QUAD $0xffffff0f0e0d0c0b + QUAD $0x0908070605040302 + QUAD $0xffff0f0e0d0c0b0a + QUAD $0x0807060504030201 + QUAD $0xff0f0e0d0c0b0a09 diff --git a/src/pkg/runtime/asm_arm.s b/src/pkg/runtime/asm_arm.s index 45b53541b..6b2d6afda 100644 --- a/src/pkg/runtime/asm_arm.s +++ b/src/pkg/runtime/asm_arm.s @@ -47,6 +47,7 @@ TEXT _rt0_arm(SB),7,$-4 MOVW R1, 8(R13) BL runtime·args(SB) BL runtime·osinit(SB) + BL runtime·hashinit(SB) BL runtime·schedinit(SB) // create a new goroutine to start program @@ -489,3 +490,17 @@ TEXT runtime·stackguard(SB),7,$0 MOVW R1, sp+0(FP) MOVW R2, limit+4(FP) RET + +// not implemented for ARM +TEXT runtime·aeshash(SB),7,$-4 + MOVW $0, R0 + MOVW (R0), R1 +TEXT runtime·aeshash32(SB),7,$-4 + MOVW $0, R0 + MOVW (R0), R1 +TEXT runtime·aeshash64(SB),7,$-4 + MOVW $0, R0 + MOVW (R0), R1 +TEXT runtime·aeshashstr(SB),7,$-4 + MOVW $0, R0 + MOVW (R0), R1 diff --git a/src/pkg/runtime/atomic_386.c b/src/pkg/runtime/atomic_386.c index 79b7cbf96..1046eb81e 100644 --- a/src/pkg/runtime/atomic_386.c +++ b/src/pkg/runtime/atomic_386.c @@ -30,3 +30,16 @@ runtime·xadd64(uint64 volatile* addr, int64 v) } return old+v; } + +#pragma textflag 7 +uint64 +runtime·xchg64(uint64 volatile* addr, uint64 v) +{ + uint64 old; + + old = *addr; + while(!runtime·cas64(addr, &old, v)) { + // nothing + } + return old; +} diff --git a/src/pkg/runtime/atomic_arm.c b/src/pkg/runtime/atomic_arm.c index 0b54840cc..9193d599d 100644 --- a/src/pkg/runtime/atomic_arm.c +++ b/src/pkg/runtime/atomic_arm.c @@ -123,6 +123,19 @@ runtime·xadd64(uint64 volatile *addr, int64 delta) #pragma textflag 7 uint64 +runtime·xchg64(uint64 volatile *addr, uint64 v) +{ + uint64 res; + + runtime·lock(LOCK(addr)); + res = *addr; + *addr = v; + runtime·unlock(LOCK(addr)); + return res; +} + +#pragma textflag 7 +uint64 runtime·atomicload64(uint64 volatile *addr) { uint64 res; diff --git a/src/pkg/runtime/cgo/callbacks.c b/src/pkg/runtime/cgo/callbacks.c index 51bd529ec..19f6115a6 100644 --- a/src/pkg/runtime/cgo/callbacks.c +++ b/src/pkg/runtime/cgo/callbacks.c @@ -12,8 +12,10 @@ // void crosscall2(void (*fn)(void *, int), void *, int); // // We need to export the symbol crosscall2 in order to support -// callbacks from shared libraries. -#pragma dynexport crosscall2 crosscall2 +// callbacks from shared libraries. This applies regardless of +// linking mode. +#pragma cgo_export_static crosscall2 +#pragma cgo_export_dynamic crosscall2 // Allocate memory. This allocates the requested number of bytes in // memory controlled by the Go runtime. The allocated memory will be diff --git a/src/pkg/runtime/cgo/gcc_freebsd_arm.c b/src/pkg/runtime/cgo/gcc_freebsd_arm.c index 3bcb0b270..73c990c28 100644 --- a/src/pkg/runtime/cgo/gcc_freebsd_arm.c +++ b/src/pkg/runtime/cgo/gcc_freebsd_arm.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include <sys/types.h> +#include <machine/sysarch.h> #include <pthread.h> #include <string.h> #include "libcgo.h" @@ -22,10 +24,20 @@ void x_cgo_load_gm(void) __attribute__((naked)); void __aeabi_read_tp(void) { - // read @ 0xffff1000 __asm__ __volatile__ ( +#ifdef ARM_TP_ADDRESS + // ARM_TP_ADDRESS is (ARM_VECTORS_HIGH + 0x1000) or 0xffff1000 + // GCC inline asm doesn't provide a way to provide a constant + // to "ldr r0, =??" pseudo instruction, so we hardcode the value + // and check it with cpp. +#if ARM_TP_ADDRESS != 0xffff1000 +#error Wrong ARM_TP_ADDRESS! +#endif "ldr r0, =0xffff1000\n\t" "ldr r0, [r0]\n\t" +#else + "mrc p15, 0, r0, c13, c0, 3\n\t" +#endif "mov pc, lr\n\t" ); } diff --git a/src/pkg/runtime/cgo/gcc_openbsd_386.c b/src/pkg/runtime/cgo/gcc_openbsd_386.c index 86c1365ad..80be31b9c 100644 --- a/src/pkg/runtime/cgo/gcc_openbsd_386.c +++ b/src/pkg/runtime/cgo/gcc_openbsd_386.c @@ -48,9 +48,9 @@ tcb_fixup(int mainthread) bcopy(oldtcb, newtcb + TLS_SIZE, TCB_SIZE); __set_tcb(newtcb + TLS_SIZE); - // The main thread TCB is a static allocation - do not try to free it. - if(!mainthread) - free(oldtcb); + // NOTE(jsing, minux): we can't free oldtcb without causing double-free + // problem. so newtcb will be memory leaks. Get rid of this when OpenBSD + // has proper support for PT_TLS. } static void * diff --git a/src/pkg/runtime/cgo/gcc_openbsd_amd64.c b/src/pkg/runtime/cgo/gcc_openbsd_amd64.c index d3a5e36b0..e9cc8184b 100644 --- a/src/pkg/runtime/cgo/gcc_openbsd_amd64.c +++ b/src/pkg/runtime/cgo/gcc_openbsd_amd64.c @@ -48,9 +48,9 @@ tcb_fixup(int mainthread) bcopy(oldtcb, newtcb + TLS_SIZE, TCB_SIZE); __set_tcb(newtcb + TLS_SIZE); - // The main thread TCB is a static allocation - do not try to free it. - if(!mainthread) - free(oldtcb); + // NOTE(jsing, minux): we can't free oldtcb without causing double-free + // problem. so newtcb will be memory leaks. Get rid of this when OpenBSD + // has proper support for PT_TLS. } static void * diff --git a/src/pkg/runtime/cgocall.c b/src/pkg/runtime/cgocall.c index 590bf9b67..0c9618749 100644 --- a/src/pkg/runtime/cgocall.c +++ b/src/pkg/runtime/cgocall.c @@ -95,7 +95,8 @@ static void unwindm(void); // Call from Go to C. -static FuncVal unlockOSThread = { runtime·unlockOSThread }; +static void endcgo(void); +static FuncVal endcgoV = { endcgo }; void runtime·cgocall(void (*fn)(void*), void *arg) @@ -123,7 +124,7 @@ runtime·cgocall(void (*fn)(void*), void *arg) * cgo callback. Add entry to defer stack in case of panic. */ runtime·lockOSThread(); - d.fn = &unlockOSThread; + d.fn = &endcgoV; d.siz = 0; d.link = g->defer; d.argp = (void*)-1; // unused because unlockm never recovers @@ -148,6 +149,16 @@ runtime·cgocall(void (*fn)(void*), void *arg) runtime·asmcgocall(fn, arg); runtime·exitsyscall(); + if(g->defer != &d || d.fn != &endcgoV) + runtime·throw("runtime: bad defer entry in cgocallback"); + g->defer = d.link; + endcgo(); +} + +static void +endcgo(void) +{ + runtime·unlockOSThread(); m->ncgo--; if(m->ncgo == 0) { // We are going back to Go and are not in a recursive @@ -156,11 +167,6 @@ runtime·cgocall(void (*fn)(void*), void *arg) m->cgomal = nil; } - if(g->defer != &d || d.fn != &unlockOSThread) - runtime·throw("runtime: bad defer entry in cgocallback"); - g->defer = d.link; - runtime·unlockOSThread(); - if(raceenabled) runtime·raceacquire(&cgosync); } @@ -280,3 +286,9 @@ runtime·cgounimpl(void) // called from (incomplete) assembly { runtime·throw("runtime: cgo not implemented"); } + +// For cgo-using programs with external linking, +// export "main" (defined in assembly) so that libc can handle basic +// C runtime startup and call the Go program as if it were +// the C main function. +#pragma cgo_export_static main diff --git a/src/pkg/runtime/crash_test.go b/src/pkg/runtime/crash_test.go index 5f84cb5a2..80549a505 100644 --- a/src/pkg/runtime/crash_test.go +++ b/src/pkg/runtime/crash_test.go @@ -91,6 +91,14 @@ func TestLockedDeadlock2(t *testing.T) { testDeadlock(t, lockedDeadlockSource2) } +func TestGoexitDeadlock(t *testing.T) { + got := executeTest(t, goexitDeadlockSource, nil) + want := "" + if got != want { + t.Fatalf("expected %q, but got %q", want, got) + } +} + const crashSource = ` package main @@ -175,3 +183,21 @@ func main() { select {} } ` + +const goexitDeadlockSource = ` +package main +import ( + "runtime" +) + +func F() { + for i := 0; i < 10; i++ { + } +} + +func main() { + go F() + go F() + runtime.Goexit() +} +` diff --git a/src/pkg/runtime/defs2_linux.go b/src/pkg/runtime/defs2_linux.go index 9b0702955..60ecc69bb 100644 --- a/src/pkg/runtime/defs2_linux.go +++ b/src/pkg/runtime/defs2_linux.go @@ -7,7 +7,7 @@ /* * Input to cgo -cdefs -GOARCH=386 cgo -cdefs defs2.go >386/defs.h +GOARCH=386 go tool cgo -cdefs defs2_linux.go >defs_linux_386.h The asm header tricks we have to use for Linux on amd64 (see defs.c and defs1.c) don't work here, so this is yet another @@ -17,15 +17,19 @@ file. Sigh. package runtime /* -#cgo CFLAGS: -I/home/rsc/pub/linux-2.6/arch/x86/include -I/home/rsc/pub/linux-2.6/include -D_LOOSE_KERNEL_NAMES -D__ARCH_SI_UID_T=__kernel_uid32_t +#cgo CFLAGS: -I/tmp/linux/arch/x86/include -I/tmp/linux/include -D_LOOSE_KERNEL_NAMES -D__ARCH_SI_UID_T=__kernel_uid32_t #define size_t __kernel_size_t +#define pid_t int #include <asm/signal.h> #include <asm/mman.h> #include <asm/sigcontext.h> #include <asm/ucontext.h> #include <asm/siginfo.h> +#include <asm-generic/errno.h> #include <asm-generic/fcntl.h> +#include <asm-generic/poll.h> +#include <linux/eventpoll.h> // This is the sigaction structure from the Linux 2.1.68 kernel which // is used with the rt_sigaction system call. For 386 this is not @@ -35,12 +39,16 @@ struct kernel_sigaction { __sighandler_t k_sa_handler; unsigned long sa_flags; void (*sa_restorer) (void); - sigset_t sa_mask; + unsigned long long sa_mask; }; */ import "C" const ( + EINTR = C.EINTR + EAGAIN = C.EAGAIN + ENOMEM = C.ENOMEM + PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ PROT_WRITE = C.PROT_WRITE @@ -110,6 +118,17 @@ const ( O_RDONLY = C.O_RDONLY O_CLOEXEC = C.O_CLOEXEC + + EPOLLIN = C.POLLIN + EPOLLOUT = C.POLLOUT + EPOLLERR = C.POLLERR + EPOLLHUP = C.POLLHUP + EPOLLRDHUP = C.POLLRDHUP + EPOLLET = C.EPOLLET + EPOLL_CLOEXEC = C.EPOLL_CLOEXEC + EPOLL_CTL_ADD = C.EPOLL_CTL_ADD + EPOLL_CTL_DEL = C.EPOLL_CTL_DEL + EPOLL_CTL_MOD = C.EPOLL_CTL_MOD ) type Fpreg C.struct__fpreg @@ -124,3 +143,4 @@ type Sigaltstack C.struct_sigaltstack type Sigcontext C.struct_sigcontext type Ucontext C.struct_ucontext type Itimerval C.struct_itimerval +type EpollEvent C.struct_epoll_event diff --git a/src/pkg/runtime/defs_darwin.go b/src/pkg/runtime/defs_darwin.go index 7f22b0b8e..722013ba9 100644 --- a/src/pkg/runtime/defs_darwin.go +++ b/src/pkg/runtime/defs_darwin.go @@ -7,8 +7,8 @@ /* Input to cgo. -GOARCH=amd64 cgo -cdefs defs_darwin.go >defs_darwin_amd64.h -GOARCH=386 cgo -cdefs defs_darwin.go >defs_darwin_386.h +GOARCH=amd64 go tool cgo -cdefs defs_darwin.go >defs_darwin_amd64.h +GOARCH=386 go tool cgo -cdefs defs_darwin.go >defs_darwin_386.h */ package runtime @@ -19,12 +19,17 @@ package runtime #include <mach/message.h> #include <sys/types.h> #include <sys/time.h> +#include <errno.h> #include <signal.h> +#include <sys/event.h> #include <sys/mman.h> */ import "C" const ( + EINTR = C.EINTR + EFAULT = C.EFAULT + PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ PROT_WRITE = C.PROT_WRITE @@ -128,6 +133,14 @@ const ( ITIMER_REAL = C.ITIMER_REAL ITIMER_VIRTUAL = C.ITIMER_VIRTUAL ITIMER_PROF = C.ITIMER_PROF + + EV_ADD = C.EV_ADD + EV_DELETE = C.EV_DELETE + EV_CLEAR = C.EV_CLEAR + EV_RECEIPT = C.EV_RECEIPT + EV_ERROR = C.EV_ERROR + EVFILT_READ = C.EVFILT_READ + EVFILT_WRITE = C.EVFILT_WRITE ) type MachBody C.mach_msg_body_t @@ -144,6 +157,7 @@ type Sigval C.union_sigval type Siginfo C.siginfo_t type Timeval C.struct_timeval type Itimerval C.struct_itimerval +type Timespec C.struct_timespec type FPControl C.struct_fp_control type FPStatus C.struct_fp_status @@ -161,3 +175,5 @@ type ExceptionState32 C.struct_i386_exception_state type Mcontext32 C.struct_mcontext32 type Ucontext C.struct_ucontext + +type Kevent C.struct_kevent diff --git a/src/pkg/runtime/defs_darwin_386.h b/src/pkg/runtime/defs_darwin_386.h index 92732f460..7b210eebf 100644 --- a/src/pkg/runtime/defs_darwin_386.h +++ b/src/pkg/runtime/defs_darwin_386.h @@ -3,6 +3,9 @@ enum { + EINTR = 0x4, + EFAULT = 0xe, + PROT_NONE = 0x0, PROT_READ = 0x1, PROT_WRITE = 0x2, @@ -106,6 +109,14 @@ enum { ITIMER_REAL = 0x0, ITIMER_VIRTUAL = 0x1, ITIMER_PROF = 0x2, + + EV_ADD = 0x1, + EV_DELETE = 0x2, + EV_CLEAR = 0x20, + EV_RECEIPT = 0x40, + EV_ERROR = 0x4000, + EVFILT_READ = -0x1, + EVFILT_WRITE = -0x2, }; typedef struct MachBody MachBody; @@ -117,6 +128,7 @@ typedef struct Sigaction Sigaction; typedef struct Siginfo Siginfo; typedef struct Timeval Timeval; typedef struct Itimerval Itimerval; +typedef struct Timespec Timespec; typedef struct FPControl FPControl; typedef struct FPStatus FPStatus; typedef struct RegMMST RegMMST; @@ -130,6 +142,7 @@ typedef struct FloatState32 FloatState32; typedef struct ExceptionState32 ExceptionState32; typedef struct Mcontext32 Mcontext32; typedef struct Ucontext Ucontext; +typedef struct Kevent Kevent; #pragma pack on @@ -170,7 +183,7 @@ struct StackT { typedef byte Sighandler[4]; struct Sigaction { - Sighandler __sigaction_u; + byte __sigaction_u[4]; void *sa_tramp; uint32 sa_mask; int32 sa_flags; @@ -185,7 +198,7 @@ struct Siginfo { uint32 si_uid; int32 si_status; byte *si_addr; - Sigval si_value; + byte si_value[4]; int32 si_band; uint32 __pad[7]; }; @@ -197,6 +210,10 @@ struct Itimerval { Timeval it_interval; Timeval it_value; }; +struct Timespec { + int32 tv_sec; + int32 tv_nsec; +}; struct FPControl { byte Pad_cgo_0[2]; @@ -362,5 +379,14 @@ struct Ucontext { Mcontext32 *uc_mcontext; }; +struct Kevent { + uint32 ident; + int16 filter; + uint16 flags; + uint32 fflags; + int32 data; + byte *udata; +}; + #pragma pack off diff --git a/src/pkg/runtime/defs_darwin_amd64.h b/src/pkg/runtime/defs_darwin_amd64.h index d4fbfef49..2d464a9e5 100644 --- a/src/pkg/runtime/defs_darwin_amd64.h +++ b/src/pkg/runtime/defs_darwin_amd64.h @@ -3,6 +3,9 @@ enum { + EINTR = 0x4, + EFAULT = 0xe, + PROT_NONE = 0x0, PROT_READ = 0x1, PROT_WRITE = 0x2, @@ -106,6 +109,14 @@ enum { ITIMER_REAL = 0x0, ITIMER_VIRTUAL = 0x1, ITIMER_PROF = 0x2, + + EV_ADD = 0x1, + EV_DELETE = 0x2, + EV_CLEAR = 0x20, + EV_RECEIPT = 0x40, + EV_ERROR = 0x4000, + EVFILT_READ = -0x1, + EVFILT_WRITE = -0x2, }; typedef struct MachBody MachBody; @@ -117,6 +128,7 @@ typedef struct Sigaction Sigaction; typedef struct Siginfo Siginfo; typedef struct Timeval Timeval; typedef struct Itimerval Itimerval; +typedef struct Timespec Timespec; typedef struct FPControl FPControl; typedef struct FPStatus FPStatus; typedef struct RegMMST RegMMST; @@ -130,6 +142,7 @@ typedef struct FloatState32 FloatState32; typedef struct ExceptionState32 ExceptionState32; typedef struct Mcontext32 Mcontext32; typedef struct Ucontext Ucontext; +typedef struct Kevent Kevent; #pragma pack on @@ -171,7 +184,7 @@ struct StackT { typedef byte Sighandler[8]; struct Sigaction { - Sighandler __sigaction_u; + byte __sigaction_u[8]; void *sa_tramp; uint32 sa_mask; int32 sa_flags; @@ -186,7 +199,7 @@ struct Siginfo { uint32 si_uid; int32 si_status; byte *si_addr; - Sigval si_value; + byte si_value[8]; int64 si_band; uint64 __pad[7]; }; @@ -199,6 +212,10 @@ struct Itimerval { Timeval it_interval; Timeval it_value; }; +struct Timespec { + int64 tv_sec; + int64 tv_nsec; +}; struct FPControl { byte Pad_cgo_0[2]; @@ -365,5 +382,14 @@ struct Ucontext { Mcontext64 *uc_mcontext; }; +struct Kevent { + uint64 ident; + int16 filter; + uint16 flags; + uint32 fflags; + int64 data; + byte *udata; +}; + #pragma pack off diff --git a/src/pkg/runtime/defs_linux.go b/src/pkg/runtime/defs_linux.go index c0275e111..2f4e03a01 100644 --- a/src/pkg/runtime/defs_linux.go +++ b/src/pkg/runtime/defs_linux.go @@ -7,7 +7,7 @@ /* Input to cgo -cdefs -GOARCH=amd64 cgo -cdefs defs.go defs1.go >amd64/defs.h +GOARCH=amd64 go tool cgo -cdefs defs_linux.go defs1_linux.go >defs_linux_amd64.h */ package runtime @@ -25,10 +25,17 @@ package runtime #include <asm/signal.h> #include <asm/siginfo.h> #include <asm/mman.h> +#include <asm-generic/errno.h> +#include <asm-generic/poll.h> +#include <linux/eventpoll.h> */ import "C" const ( + EINTR = C.EINTR + EAGAIN = C.EAGAIN + ENOMEM = C.ENOMEM + PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ PROT_WRITE = C.PROT_WRITE @@ -95,6 +102,17 @@ const ( ITIMER_REAL = C.ITIMER_REAL ITIMER_VIRTUAL = C.ITIMER_VIRTUAL ITIMER_PROF = C.ITIMER_PROF + + EPOLLIN = C.POLLIN + EPOLLOUT = C.POLLOUT + EPOLLERR = C.POLLERR + EPOLLHUP = C.POLLHUP + EPOLLRDHUP = C.POLLRDHUP + EPOLLET = C.EPOLLET + EPOLL_CLOEXEC = C.EPOLL_CLOEXEC + EPOLL_CTL_ADD = C.EPOLL_CTL_ADD + EPOLL_CTL_DEL = C.EPOLL_CTL_DEL + EPOLL_CTL_MOD = C.EPOLL_CTL_MOD ) type Timespec C.struct_timespec @@ -102,3 +120,4 @@ type Timeval C.struct_timeval type Sigaction C.struct_sigaction type Siginfo C.siginfo_t type Itimerval C.struct_itimerval +type EpollEvent C.struct_epoll_event diff --git a/src/pkg/runtime/defs_linux_386.h b/src/pkg/runtime/defs_linux_386.h index e257a6f85..27dae9e82 100644 --- a/src/pkg/runtime/defs_linux_386.h +++ b/src/pkg/runtime/defs_linux_386.h @@ -1,8 +1,12 @@ // Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs2.go +// cgo -cdefs defs2_linux.go enum { + EINTR = 0x4, + EAGAIN = 0xb, + ENOMEM = 0xc, + PROT_NONE = 0x0, PROT_READ = 0x1, PROT_WRITE = 0x2, @@ -72,6 +76,17 @@ enum { O_RDONLY = 0x0, O_CLOEXEC = 0x80000, + + EPOLLIN = 0x1, + EPOLLOUT = 0x4, + EPOLLERR = 0x8, + EPOLLHUP = 0x10, + EPOLLRDHUP = 0x2000, + EPOLLET = -0x80000000, + EPOLL_CLOEXEC = 0x80000, + EPOLL_CTL_ADD = 0x1, + EPOLL_CTL_DEL = 0x2, + EPOLL_CTL_MOD = 0x3, }; typedef struct Fpreg Fpreg; @@ -86,6 +101,7 @@ typedef struct Sigaltstack Sigaltstack; typedef struct Sigcontext Sigcontext; typedef struct Ucontext Ucontext; typedef struct Itimerval Itimerval; +typedef struct EpollEvent EpollEvent; #pragma pack on @@ -186,6 +202,10 @@ struct Itimerval { Timeval it_interval; Timeval it_value; }; +struct EpollEvent { + uint32 events; + uint64 data; +}; #pragma pack off diff --git a/src/pkg/runtime/defs_linux_amd64.h b/src/pkg/runtime/defs_linux_amd64.h index bf5f79b0e..3e87df68a 100644 --- a/src/pkg/runtime/defs_linux_amd64.h +++ b/src/pkg/runtime/defs_linux_amd64.h @@ -1,8 +1,12 @@ // Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs.go defs1.go +// cgo -cdefs defs_linux.go defs1_linux.go enum { + EINTR = 0x4, + EAGAIN = 0xb, + ENOMEM = 0xc, + PROT_NONE = 0x0, PROT_READ = 0x1, PROT_WRITE = 0x2, @@ -69,6 +73,17 @@ enum { ITIMER_REAL = 0x0, ITIMER_VIRTUAL = 0x1, ITIMER_PROF = 0x2, + + EPOLLIN = 0x1, + EPOLLOUT = 0x4, + EPOLLERR = 0x8, + EPOLLHUP = 0x10, + EPOLLRDHUP = 0x2000, + EPOLLET = -0x80000000, + EPOLL_CLOEXEC = 0x80000, + EPOLL_CTL_ADD = 0x1, + EPOLL_CTL_DEL = 0x2, + EPOLL_CTL_MOD = 0x3, }; typedef struct Timespec Timespec; @@ -76,6 +91,7 @@ typedef struct Timeval Timeval; typedef struct Sigaction Sigaction; typedef struct Siginfo Siginfo; typedef struct Itimerval Itimerval; +typedef struct EpollEvent EpollEvent; #pragma pack on @@ -104,11 +120,15 @@ struct Itimerval { Timeval it_interval; Timeval it_value; }; +struct EpollEvent { + uint32 events; + uint64 data; +}; #pragma pack off // Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs.go defs1.go +// cgo -cdefs defs_linux.go defs1_linux.go enum { diff --git a/src/pkg/runtime/defs_linux_arm.h b/src/pkg/runtime/defs_linux_arm.h index f72ec3d1b..92160966e 100644 --- a/src/pkg/runtime/defs_linux_arm.h +++ b/src/pkg/runtime/defs_linux_arm.h @@ -1,9 +1,11 @@ -// godefs -f-I/usr/src/linux-headers-2.6.26-2-versatile/include defs_arm.c - -// MACHINE GENERATED - DO NOT EDIT. +// TODO: Generate using cgo like defs_linux_{386,amd64}.h // Constants enum { + EINTR = 0x4, + ENOMEM = 0xc, + EAGAIN = 0xb, + PROT_NONE = 0, PROT_READ = 0x1, PROT_WRITE = 0x2, @@ -64,6 +66,17 @@ enum { ITIMER_VIRTUAL = 0x1, O_RDONLY = 0, O_CLOEXEC = 02000000, + + EPOLLIN = 0x1, + EPOLLOUT = 0x4, + EPOLLERR = 0x8, + EPOLLHUP = 0x10, + EPOLLRDHUP = 0x2000, + EPOLLET = -0x80000000, + EPOLL_CLOEXEC = 0x80000, + EPOLL_CTL_ADD = 0x1, + EPOLL_CTL_DEL = 0x2, + EPOLL_CTL_MOD = 0x3, }; // Types @@ -145,4 +158,11 @@ struct Sigaction { void *sa_restorer; uint64 sa_mask; }; + +typedef struct EpollEvent EpollEvent; +struct EpollEvent { + uint32 events; + uint32 _pad; + uint64 data; +}; #pragma pack off diff --git a/src/pkg/runtime/defs_netbsd.go b/src/pkg/runtime/defs_netbsd.go index 53e061041..c543593fa 100644 --- a/src/pkg/runtime/defs_netbsd.go +++ b/src/pkg/runtime/defs_netbsd.go @@ -9,6 +9,7 @@ Input to cgo. GOARCH=amd64 go tool cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go >defs_netbsd_amd64.h GOARCH=386 go tool cgo -cdefs defs_netbsd.go defs_netbsd_386.go >defs_netbsd_386.h +GOARCH=arm go tool cgo -cdefs defs_netbsd.go defs_netbsd_arm.go >defs_netbsd_arm.h */ // +godefs map __fpregset_t [644]byte diff --git a/src/pkg/runtime/defs_netbsd_386.go b/src/pkg/runtime/defs_netbsd_386.go index e9e36608e..c26f24607 100644 --- a/src/pkg/runtime/defs_netbsd_386.go +++ b/src/pkg/runtime/defs_netbsd_386.go @@ -7,7 +7,6 @@ /* Input to cgo. -GOARCH=amd64 go tool cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go >defs_netbsd_amd64.h GOARCH=386 go tool cgo -cdefs defs_netbsd.go defs_netbsd_386.go >defs_netbsd_386.h */ diff --git a/src/pkg/runtime/defs_netbsd_amd64.go b/src/pkg/runtime/defs_netbsd_amd64.go index 68f586b2f..f18a7b1fe 100644 --- a/src/pkg/runtime/defs_netbsd_amd64.go +++ b/src/pkg/runtime/defs_netbsd_amd64.go @@ -8,7 +8,6 @@ Input to cgo. GOARCH=amd64 go tool cgo -cdefs defs_netbsd.go defs_netbsd_amd64.go >defs_netbsd_amd64.h -GOARCH=386 go tool cgo -cdefs defs_netbsd.go defs_netbsd_386.go >defs_netbsd_386.h */ package runtime diff --git a/src/pkg/runtime/defs_netbsd_arm.go b/src/pkg/runtime/defs_netbsd_arm.go new file mode 100644 index 000000000..cb0dce66b --- /dev/null +++ b/src/pkg/runtime/defs_netbsd_arm.go @@ -0,0 +1,39 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo. + +GOARCH=arm go tool cgo -cdefs defs_netbsd.go defs_netbsd_arm.go >defs_netbsd_arm.h +*/ + +package runtime + +/* +#include <sys/types.h> +#include <machine/mcontext.h> +*/ +import "C" + +const ( + REG_R0 = C._REG_R0 + REG_R1 = C._REG_R1 + REG_R2 = C._REG_R2 + REG_R3 = C._REG_R3 + REG_R4 = C._REG_R4 + REG_R5 = C._REG_R5 + REG_R6 = C._REG_R6 + REG_R7 = C._REG_R7 + REG_R8 = C._REG_R8 + REG_R9 = C._REG_R9 + REG_R10 = C._REG_R10 + REG_R11 = C._REG_R11 + REG_R12 = C._REG_R12 + REG_R13 = C._REG_R13 + REG_R14 = C._REG_R14 + REG_R15 = C._REG_R15 + REG_CPSR = C._REG_CPSR +) diff --git a/src/pkg/runtime/defs_netbsd_arm.h b/src/pkg/runtime/defs_netbsd_arm.h index f67475c76..26b55222e 100644 --- a/src/pkg/runtime/defs_netbsd_arm.h +++ b/src/pkg/runtime/defs_netbsd_arm.h @@ -1,5 +1,5 @@ // Created by cgo -cdefs - DO NOT EDIT -// cgo -cdefs defs_netbsd.go +// cgo -cdefs defs_netbsd.go defs_netbsd_arm.go enum { @@ -138,3 +138,27 @@ struct UcontextT { }; #pragma pack off +// Created by cgo -cdefs - DO NOT EDIT +// cgo -cdefs defs_netbsd.go defs_netbsd_arm.go + + +enum { + REG_R0 = 0x0, + REG_R1 = 0x1, + REG_R2 = 0x2, + REG_R3 = 0x3, + REG_R4 = 0x4, + REG_R5 = 0x5, + REG_R6 = 0x6, + REG_R7 = 0x7, + REG_R8 = 0x8, + REG_R9 = 0x9, + REG_R10 = 0xa, + REG_R11 = 0xb, + REG_R12 = 0xc, + REG_R13 = 0xd, + REG_R14 = 0xe, + REG_R15 = 0xf, + REG_CPSR = 0x10, +}; + diff --git a/src/pkg/runtime/env_plan9.c b/src/pkg/runtime/env_plan9.c index 848d73303..0483d7eef 100644 --- a/src/pkg/runtime/env_plan9.c +++ b/src/pkg/runtime/env_plan9.c @@ -20,7 +20,7 @@ runtime·getenv(int8 *s) runtime·memmove((void*)file, (void*)"/env/", 5); runtime·memmove((void*)(file+5), (void*)s, len); - fd = runtime·open(file, OREAD); + fd = runtime·open((int8*)file, OREAD, 0); if(fd < 0) return nil; n = runtime·seek(fd, 0, 2); diff --git a/src/pkg/runtime/export_futex_test.go b/src/pkg/runtime/export_futex_test.go new file mode 100644 index 000000000..bcab60fbe --- /dev/null +++ b/src/pkg/runtime/export_futex_test.go @@ -0,0 +1,13 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux freebsd + +package runtime + +func futexsleep(addr *uint32, val uint32, ns int64) +func futexwakeup(addr *uint32, val uint32) + +var Futexsleep = futexsleep +var Futexwakeup = futexwakeup diff --git a/src/pkg/runtime/extern.go b/src/pkg/runtime/extern.go index fbaffd1d5..20f234253 100644 --- a/src/pkg/runtime/extern.go +++ b/src/pkg/runtime/extern.go @@ -3,10 +3,54 @@ // license that can be found in the LICENSE file. /* - Package runtime contains operations that interact with Go's runtime system, - such as functions to control goroutines. It also includes the low-level type information - used by the reflect package; see reflect's documentation for the programmable - interface to the run-time type system. +Package runtime contains operations that interact with Go's runtime system, +such as functions to control goroutines. It also includes the low-level type information +used by the reflect package; see reflect's documentation for the programmable +interface to the run-time type system. + +Environment Variables + +The following environment variables ($name or %name%, depending on the host +operating system) control the run-time behavior of Go programs. The meanings +and use may change from release to release. + +The GOGC variable sets the initial garbage collection target percentage. +A collection is triggered when the ratio of freshly allocated data to live data +remaining after the previous collection reaches this percentage. The default +is GOGC=100. Setting GOGC=off disables the garbage collector entirely. +The runtime/debug package's SetGCPercent function allows changing this +percentage at run time. See http://golang.org/pkg/runtime/debug/#SetGCPercent. + +The GOGCTRACE variable controls debug output from the garbage collector. +Setting GOGCTRACE=1 causes the garbage collector to emit a single line to standard +error at each collection, summarizing the amount of memory collected and the +length of the pause. Setting GOGCTRACE=2 emits the same summary but also +repeats each collection. + +The GOMAXPROCS variable limits the number of operating system threads that +can execute user-level Go code simultaneously. There is no limit to the number of threads +that can be blocked in system calls on behalf of Go code; those do not count against +the GOMAXPROCS limit. This package's GOMAXPROCS function queries and changes +the limit. + +The GOTRACEBACK variable controls the amount of output generated when a Go +program fails due to an unrecovered panic or an unexpected runtime condition. +By default, a failure prints a stack trace for every extant goroutine, eliding functions +internal to the run-time system, and then exits with exit code 2. +If GOTRACEBACK=0, the per-goroutine stack traces are omitted entirely. +If GOTRACEBACK=1, the default behavior is used. +If GOTRACEBACK=2, the per-goroutine stack traces include run-time functions. +If GOTRACEBACK=crash, the per-goroutine stack traces include run-time functions, +and if possible the program crashes in an operating-specific manner instead of +exiting. For example, on Unix systems, the program raises SIGABRT to trigger a +core dump. + +The GOARCH, GOOS, GOPATH, and GOROOT environment variables complete +the set of Go environment variables. They influence the building of Go programs +(see http://golang.org/cmd/go and http://golang.org/pkg/go/build). +GOARCH, GOOS, and GOROOT are recorded at compile time and made available by +constants or functions in this package, but they do not influence the execution +of the run-time system. */ package runtime diff --git a/src/pkg/runtime/futex_test.go b/src/pkg/runtime/futex_test.go new file mode 100644 index 000000000..51f4d0f12 --- /dev/null +++ b/src/pkg/runtime/futex_test.go @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux freebsd + +package runtime_test + +import ( + . "runtime" + "testing" + "time" +) + +func TestFutexsleep(t *testing.T) { + ch := make(chan bool, 1) + var dummy uint32 + start := time.Now() + go func() { + Entersyscall() + Futexsleep(&dummy, 0, (1<<31+100)*1e9) + Exitsyscall() + ch <- true + }() + select { + case <-ch: + t.Errorf("futexsleep finished early after %s!", time.Since(start)) + case <-time.After(time.Second): + Futexwakeup(&dummy, 1) + } +} diff --git a/src/pkg/runtime/gc_test.go b/src/pkg/runtime/gc_test.go index e1e1b1d01..3475339bf 100644 --- a/src/pkg/runtime/gc_test.go +++ b/src/pkg/runtime/gc_test.go @@ -7,6 +7,7 @@ package runtime_test import ( "os" "runtime" + "runtime/debug" "testing" ) @@ -82,3 +83,17 @@ func TestGcDeepNesting(t *testing.T) { t.Fail() } } + +func TestGcHashmapIndirection(t *testing.T) { + defer debug.SetGCPercent(debug.SetGCPercent(1)) + runtime.GC() + type T struct { + a [256]int + } + m := make(map[T]T) + for i := 0; i < 2000; i++ { + var a T + a.a[0] = i + m[a] = T{} + } +} diff --git a/src/pkg/runtime/hashmap.c b/src/pkg/runtime/hashmap.c index 37111daa9..0f92becab 100644 --- a/src/pkg/runtime/hashmap.c +++ b/src/pkg/runtime/hashmap.c @@ -9,703 +9,831 @@ #include "type.h" #include "race.h" -/* Hmap flag values */ -#define IndirectVal (1<<0) /* storing pointers to values */ -#define IndirectKey (1<<1) /* storing pointers to keys */ -#define CanFreeTable (1<<2) /* okay to free subtables */ -#define CanFreeKey (1<<3) /* okay to free pointers to keys */ - -struct Hmap { /* a hash table; initialize with hash_init() */ - uintgo count; /* elements in table - must be first */ - uint8 datasize; /* amount of data to store in entry */ - uint8 flag; - uint8 valoff; /* offset of value in key+value data block */ - int32 changes; /* inc'ed whenever a subtable is created/grown */ - uintptr hash0; /* hash seed */ - struct hash_subtable *st; /* first-level table */ +// This file contains the implementation of Go's map type. +// +// The map is just a hash table. The data is arranged +// into an array of buckets. Each bucket contains up to +// 8 key/value pairs. The low-order bits of the hash are +// used to select a bucket. Each bucket contains a few +// high-order bits of each hash to distinguish the entries +// within a single bucket. +// +// If more than 8 keys hash to a bucket, we chain on +// extra buckets. +// +// When the hashtable grows, we allocate a new array +// of buckets twice as big. Buckets are incrementally +// copied from the old bucket array to the new bucket array. +// +// Map iterators walk through the array of buckets and +// return the keys in walk order (bucket #, then overflow +// chain order, then bucket index). To maintain iteration +// semantics, we never move keys within their bucket (if +// we did, keys might be returned 0 or 2 times). When +// growing the table, iterators remain iterating through the +// old table and must check the new table if the bucket +// they are iterating through has been moved ("evacuated") +// to the new table. + +// Maximum number of key/value pairs a bucket can hold. +#define BUCKETSIZE 8 + +// Maximum average load of a bucket that triggers growth. +#define LOAD 6.5 + +// Picking LOAD: too large and we have lots of overflow +// buckets, too small and we waste a lot of space. I wrote +// a simple program to check some stats for different loads: +// (64-bit, 8 byte keys and values) +// LOAD %overflow bytes/entry hitprobe missprobe +// 4.00 2.13 20.77 3.00 4.00 +// 4.50 4.05 17.30 3.25 4.50 +// 5.00 6.85 14.77 3.50 5.00 +// 5.50 10.55 12.94 3.75 5.50 +// 6.00 15.27 11.67 4.00 6.00 +// 6.50 20.90 10.79 4.25 6.50 +// 7.00 27.14 10.15 4.50 7.00 +// 7.50 34.03 9.73 4.75 7.50 +// 8.00 41.10 9.40 5.00 8.00 +// +// %overflow = percentage of buckets which have an overflow bucket +// bytes/entry = overhead bytes used per key/value pair +// hitprobe = # of entries to check when looking up a present key +// missprobe = # of entries to check when looking up an absent key +// +// Keep in mind this data is for maximally loaded tables, i.e. just +// before the table grows. Typical tables will be somewhat less loaded. + +// Maximum key or value size to keep inline (instead of mallocing per element). +// Must fit in a uint8. +// Fast versions cannot handle big values - the cutoff size for +// fast versions in ../../cmd/gc/walk.c must be at most this value. +#define MAXKEYSIZE 128 +#define MAXVALUESIZE 128 + +typedef struct Bucket Bucket; +struct Bucket +{ + uint8 tophash[BUCKETSIZE]; // top 8 bits of hash of each entry (0 = empty) + Bucket *overflow; // overflow bucket, if any + byte data[1]; // BUCKETSIZE keys followed by BUCKETSIZE values }; +// NOTE: packing all the keys together and then all the values together makes the +// code a bit more complicated than alternating key/value/key/value/... but it allows +// us to eliminate padding which would be needed for, e.g., map[int64]int8. -#define MaxData 255 +// Low-order bit of overflow field is used to mark a bucket as already evacuated +// without destroying the overflow pointer. +// Only buckets in oldbuckets will be marked as evacuated. +// Evacuated bit will be set identically on the base bucket and any overflow buckets. +#define evacuated(b) (((uintptr)(b)->overflow & 1) != 0) +#define overflowptr(b) ((Bucket*)((uintptr)(b)->overflow & ~(uintptr)1)) -struct hash_entry { - hash_hash_t hash; /* hash value of data */ - byte data[1]; /* user data has "datasize" bytes */ -}; +// Initialize bucket to the empty state. This only works if BUCKETSIZE==8! +#define clearbucket(b) { *(uint64*)((b)->tophash) = 0; (b)->overflow = nil; } -struct hash_subtable { - uint8 power; /* bits used to index this table */ - uint8 used; /* bits in hash used before reaching this table */ - uint8 datasize; /* bytes of client data in an entry */ - uint8 max_probes; /* max number of probes when searching */ - int16 limit_bytes; /* max_probes * (datasize+sizeof (hash_hash_t)) */ - struct hash_entry *last; /* points to last element of entry[] */ - struct hash_entry entry[1]; /* 2**power+max_probes-1 elements of elemsize bytes */ +struct Hmap +{ + uintgo count; // # live cells == size of map. Must be first (used by len() builtin) + uint8 B; // log_2 of # of buckets (can hold up to LOAD * 2^B items) + uint8 flags; + uint8 keysize; // key size in bytes + uint8 valuesize; // value size in bytes + uint16 bucketsize; // bucket size in bytes + + uintptr hash0; // hash seed + byte *buckets; // array of 2^B Buckets + byte *oldbuckets; // previous bucket array of half the size, non-nil only when growing + uintptr nevacuate; // progress counter for evacuation (buckets less than this have been evacuated) }; -#define HASH_DATA_EQ(eq, t, h,x,y) ((eq)=0, (*t->key->alg->equal) (&(eq), t->key->size, (x), (y)), (eq)) - -#define HASH_REHASH 0x2 /* an internal flag */ -/* the number of bits used is stored in the flags word too */ -#define HASH_USED(x) ((x) >> 2) -#define HASH_MAKE_USED(x) ((x) << 2) +// possible flags +enum +{ + IndirectKey = 1, // storing pointers to keys + IndirectValue = 2, // storing pointers to values + Iterator = 4, // there may be an iterator using buckets + OldIterator = 8, // there may be an iterator using oldbuckets + CanFreeBucket = 16, // ok to free buckets + CanFreeKey = 32, // keys are indirect and ok to free keys +}; -#define HASH_LOW 6 -#define HASH_ONE (((hash_hash_t)1) << HASH_LOW) -#define HASH_MASK (HASH_ONE - 1) -#define HASH_ADJUST(x) (((x) < HASH_ONE) << HASH_LOW) +// Macros for dereferencing indirect keys +#define IK(h, p) (((h)->flags & IndirectKey) != 0 ? *(byte**)(p) : (p)) +#define IV(h, p) (((h)->flags & IndirectValue) != 0 ? *(byte**)(p) : (p)) -#define HASH_BITS (sizeof (hash_hash_t) * 8) +enum +{ + docheck = 0, // check invariants before and after every op. Slow!!! + debug = 0, // print every operation +}; +static void +check(MapType *t, Hmap *h) +{ + uintptr bucket, oldbucket; + Bucket *b; + uintptr i; + uintptr hash; + uintgo cnt; + uint8 top; + bool eq; + byte *k, *v; -#define HASH_SUBHASH HASH_MASK -#define HASH_NIL 0 -#define HASH_NIL_MEMSET 0 + cnt = 0; -#define HASH_OFFSET(base, byte_offset) \ - ((struct hash_entry *) (((byte *) (base)) + (byte_offset))) + // check buckets + for(bucket = 0; bucket < (uintptr)1 << h->B; bucket++) { + if(h->oldbuckets != nil) { + oldbucket = bucket & (((uintptr)1 << (h->B - 1)) - 1); + b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize); + if(!evacuated(b)) + continue; // b is still uninitialized + } + for(b = (Bucket*)(h->buckets + bucket * h->bucketsize); b != nil; b = b->overflow) { + for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] == 0) + continue; + cnt++; + t->key->alg->equal(&eq, t->key->size, IK(h, k), IK(h, k)); + if(!eq) + continue; // NaN! + hash = h->hash0; + t->key->alg->hash(&hash, t->key->size, IK(h, k)); + top = hash >> (8*sizeof(uintptr) - 8); + if(top == 0) + top = 1; + if(top != b->tophash[i]) + runtime·throw("bad hash"); + } + } + } -#define HASH_MAX_PROBES 15 /* max entries to probe before rehashing */ -#define HASH_MAX_POWER 12 /* max power of 2 to create sub-tables */ + // check oldbuckets + if(h->oldbuckets != nil) { + for(oldbucket = 0; oldbucket < (uintptr)1 << (h->B - 1); oldbucket++) { + b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize); + if(evacuated(b)) + continue; + if(oldbucket < h->nevacuate) + runtime·throw("bucket became unevacuated"); + for(; b != nil; b = overflowptr(b)) { + for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] == 0) + continue; + cnt++; + t->key->alg->equal(&eq, t->key->size, IK(h, k), IK(h, k)); + if(!eq) + continue; // NaN! + hash = h->hash0; + t->key->alg->hash(&hash, t->key->size, IK(h, k)); + top = hash >> (8*sizeof(uintptr) - 8); + if(top == 0) + top = 1; + if(top != b->tophash[i]) + runtime·throw("bad hash (old)"); + } + } + } + } -/* return a hash layer with 2**power empty entries */ -static struct hash_subtable * -hash_subtable_new (Hmap *h, int32 power, int32 used) -{ - int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - int32 bytes = elemsize << power; - struct hash_subtable *st; - int32 limit_bytes = HASH_MAX_PROBES * elemsize; - int32 max_probes = HASH_MAX_PROBES; - - if (bytes < limit_bytes) { - limit_bytes = bytes; - max_probes = 1 << power; + if(cnt != h->count) { + runtime·printf("%D %D\n", (uint64)cnt, (uint64)h->count); + runtime·throw("entries missing"); } - bytes += limit_bytes - elemsize; - st = runtime·mallocgc(offsetof (struct hash_subtable, entry[0]) + bytes, UseSpanType ? FlagNoPointers : 0, 1, 1); - st->power = power; - st->used = used; - st->datasize = h->datasize; - st->max_probes = max_probes; - st->limit_bytes = limit_bytes; - st->last = HASH_OFFSET (st->entry, bytes) - 1; - memset (st->entry, HASH_NIL_MEMSET, bytes); - return (st); } static void -init_sizes (int64 hint, int32 *init_power) +hash_init(MapType *t, Hmap *h, uint32 hint) { - int32 log = 0; - int32 i; - - for (i = 32; i != 0; i >>= 1) { - if ((hint >> (log + i)) != 0) { - log += i; - } + uint8 B; + byte *buckets; + uintptr i; + uintptr keysize, valuesize, bucketsize; + uint8 flags; + Bucket *b; + + flags = CanFreeBucket; + + // figure out how big we have to make everything + keysize = t->key->size; + if(keysize > MAXKEYSIZE) { + flags |= IndirectKey | CanFreeKey; + keysize = sizeof(byte*); } - log += 1 + (((hint << 3) >> log) >= 11); /* round up for utilization */ - if (log <= 14) { - *init_power = log; - } else { - *init_power = 12; + valuesize = t->elem->size; + if(valuesize >= MAXVALUESIZE) { + flags |= IndirectValue; + valuesize = sizeof(byte*); + } + bucketsize = offsetof(Bucket, data[0]) + (keysize + valuesize) * BUCKETSIZE; + + // invariants we depend on. We should probably check these at compile time + // somewhere, but for now we'll do it here. + if(t->key->align > BUCKETSIZE) + runtime·throw("key align too big"); + if(t->elem->align > BUCKETSIZE) + runtime·throw("value align too big"); + if(t->key->size % t->key->align != 0) + runtime·throw("key size not a multiple of key align"); + if(t->elem->size % t->elem->align != 0) + runtime·throw("value size not a multiple of value align"); + if(BUCKETSIZE < 8) + runtime·throw("bucketsize too small for proper alignment"); + if(BUCKETSIZE != 8) + runtime·throw("must redo clearbucket"); + if(sizeof(void*) == 4 && t->key->align > 4) + runtime·throw("need padding in bucket (key)"); + if(sizeof(void*) == 4 && t->elem->align > 4) + runtime·throw("need padding in bucket (value)"); + + // find size parameter which will hold the requested # of elements + B = 0; + while(hint > BUCKETSIZE && hint > LOAD * ((uintptr)1 << B)) + B++; + + // allocate initial hash table + // If hint is large zeroing this memory could take a while. + buckets = runtime·mallocgc(bucketsize << B, 0, 1, 0); + for(i = 0; i < (uintptr)1 << B; i++) { + b = (Bucket*)(buckets + i * bucketsize); + clearbucket(b); } -} -static void -hash_init (Hmap *h, int32 datasize, int64 hint) -{ - int32 init_power; - - if(datasize < sizeof (void *)) - datasize = sizeof (void *); - datasize = ROUND(datasize, sizeof (void *)); - init_sizes (hint, &init_power); - h->datasize = datasize; - assert (h->datasize == datasize); - assert (sizeof (void *) <= h->datasize); + // initialize Hmap + // Note: we save all these stores to the end so gciter doesn't see + // a partially initialized map. h->count = 0; - h->changes = 0; - h->st = hash_subtable_new (h, init_power, 0); + h->B = B; + h->flags = flags; + h->keysize = keysize; + h->valuesize = valuesize; + h->bucketsize = bucketsize; h->hash0 = runtime·fastrand1(); + h->buckets = buckets; + h->oldbuckets = nil; + h->nevacuate = 0; + if(docheck) + check(t, h); } +// Moves entries in oldbuckets[i] to buckets[i] and buckets[i+2^k]. +// We leave the original bucket intact, except for the evacuated marks, so that +// iterators can still iterate through the old buckets. static void -hash_remove_n (struct hash_subtable *st, struct hash_entry *dst_e, int32 n) +evacuate(MapType *t, Hmap *h, uintptr oldbucket) { - int32 elemsize = st->datasize + offsetof (struct hash_entry, data[0]); - struct hash_entry *src_e = HASH_OFFSET (dst_e, n * elemsize); - struct hash_entry *last_e = st->last; - int32 shift = HASH_BITS - (st->power + st->used); - int32 index_mask = (((hash_hash_t)1) << st->power) - 1; - int32 dst_i = (((byte *) dst_e) - ((byte *) st->entry)) / elemsize; - int32 src_i = dst_i + n; - hash_hash_t hash; - int32 skip; - int32 bytes; - - while (dst_e != src_e) { - if (src_e <= last_e) { - struct hash_entry *cp_e = src_e; - int32 save_dst_i = dst_i; - while (cp_e <= last_e && (hash = cp_e->hash) != HASH_NIL && - ((hash >> shift) & index_mask) <= dst_i) { - cp_e = HASH_OFFSET (cp_e, elemsize); - dst_i++; + Bucket *b; + Bucket *nextb; + Bucket *x, *y; + Bucket *newx, *newy; + uintptr xi, yi; + uintptr newbit; + uintptr hash; + uintptr i; + byte *k, *v; + byte *xk, *yk, *xv, *yv; + byte *ob; + + b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize); + newbit = (uintptr)1 << (h->B - 1); + + if(!evacuated(b)) { + // TODO: reuse overflow buckets instead of using new ones, if there + // is no iterator using the old buckets. (If CanFreeBuckets and !OldIterator.) + + x = (Bucket*)(h->buckets + oldbucket * h->bucketsize); + y = (Bucket*)(h->buckets + (oldbucket + newbit) * h->bucketsize); + clearbucket(x); + clearbucket(y); + xi = 0; + yi = 0; + xk = x->data; + yk = y->data; + xv = xk + h->keysize * BUCKETSIZE; + yv = yk + h->keysize * BUCKETSIZE; + do { + for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] == 0) + continue; + hash = h->hash0; + t->key->alg->hash(&hash, t->key->size, IK(h, k)); + // NOTE: if key != key, then this hash could be (and probably will be) + // entirely different from the old hash. We effectively only update + // the B'th bit of the hash in this case. + if((hash & newbit) == 0) { + if(xi == BUCKETSIZE) { + newx = runtime·mallocgc(h->bucketsize, 0, 1, 0); + clearbucket(newx); + x->overflow = newx; + x = newx; + xi = 0; + xk = x->data; + xv = xk + h->keysize * BUCKETSIZE; + } + x->tophash[xi] = b->tophash[i]; + if((h->flags & IndirectKey) != 0) { + *(byte**)xk = *(byte**)k; // copy pointer + } else { + t->key->alg->copy(t->key->size, xk, k); // copy value + } + if((h->flags & IndirectValue) != 0) { + *(byte**)xv = *(byte**)v; + } else { + t->elem->alg->copy(t->elem->size, xv, v); + } + xi++; + xk += h->keysize; + xv += h->valuesize; + } else { + if(yi == BUCKETSIZE) { + newy = runtime·mallocgc(h->bucketsize, 0, 1, 0); + clearbucket(newy); + y->overflow = newy; + y = newy; + yi = 0; + yk = y->data; + yv = yk + h->keysize * BUCKETSIZE; + } + y->tophash[yi] = b->tophash[i]; + if((h->flags & IndirectKey) != 0) { + *(byte**)yk = *(byte**)k; + } else { + t->key->alg->copy(t->key->size, yk, k); + } + if((h->flags & IndirectValue) != 0) { + *(byte**)yv = *(byte**)v; + } else { + t->elem->alg->copy(t->elem->size, yv, v); + } + yi++; + yk += h->keysize; + yv += h->valuesize; + } } - bytes = ((byte *) cp_e) - (byte *) src_e; - memmove (dst_e, src_e, bytes); - dst_e = HASH_OFFSET (dst_e, bytes); - src_e = cp_e; - src_i += dst_i - save_dst_i; - if (src_e <= last_e && (hash = src_e->hash) != HASH_NIL) { - skip = ((hash >> shift) & index_mask) - dst_i; + + // mark as evacuated so we don't do it again. + // this also tells any iterators that this data isn't golden anymore. + nextb = b->overflow; + b->overflow = (Bucket*)((uintptr)nextb + 1); + + b = nextb; + } while(b != nil); + + // Free old overflow buckets as much as we can. + if((h->flags & OldIterator) == 0) { + b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize); + if((h->flags & CanFreeBucket) != 0) { + while((nextb = overflowptr(b)) != nil) { + b->overflow = nextb->overflow; + runtime·free(nextb); + } } else { - skip = src_i - dst_i; + // can't explicitly free overflow buckets, but at least + // we can unlink them. + b->overflow = (Bucket*)1; } - } else { - skip = src_i - dst_i; } - bytes = skip * elemsize; - memset (dst_e, HASH_NIL_MEMSET, bytes); - dst_e = HASH_OFFSET (dst_e, bytes); - dst_i += skip; } -} -static int32 -hash_insert_internal (MapType*, struct hash_subtable **pst, int32 flags, hash_hash_t hash, - Hmap *h, void *data, void **pres); + // advance evacuation mark + if(oldbucket == h->nevacuate) { + h->nevacuate = oldbucket + 1; + if(oldbucket + 1 == newbit) { // newbit == # of oldbuckets + // free main bucket array + if((h->flags & (OldIterator | CanFreeBucket)) == CanFreeBucket) { + ob = h->oldbuckets; + h->oldbuckets = nil; + runtime·free(ob); + } else { + h->oldbuckets = nil; + } + } + } + if(docheck) + check(t, h); +} static void -hash_conv (MapType *t, Hmap *h, - struct hash_subtable *st, int32 flags, - hash_hash_t hash, - struct hash_entry *e) +grow_work(MapType *t, Hmap *h, uintptr bucket) { - int32 new_flags = (flags + HASH_MAKE_USED (st->power)) | HASH_REHASH; - int32 shift = HASH_BITS - HASH_USED (new_flags); - hash_hash_t prefix_mask = (-(hash_hash_t)1) << shift; - int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - void *dummy_result; - struct hash_entry *de; - int32 index_mask = (1 << st->power) - 1; - hash_hash_t e_hash; - struct hash_entry *pe = HASH_OFFSET (e, -elemsize); - - while (e != st->entry && (e_hash = pe->hash) != HASH_NIL && (e_hash & HASH_MASK) != HASH_SUBHASH) { - e = pe; - pe = HASH_OFFSET (pe, -elemsize); - } + uintptr noldbuckets; - de = e; - while (e <= st->last && - (e_hash = e->hash) != HASH_NIL && - (e_hash & HASH_MASK) != HASH_SUBHASH) { - struct hash_entry *target_e = HASH_OFFSET (st->entry, ((e_hash >> shift) & index_mask) * elemsize); - struct hash_entry *ne = HASH_OFFSET (e, elemsize); - hash_hash_t current = e_hash & prefix_mask; - if (de < target_e) { - memset (de, HASH_NIL_MEMSET, ((byte *) target_e) - (byte *) de); - de = target_e; - } - if ((hash & prefix_mask) == current || - (ne <= st->last && (e_hash = ne->hash) != HASH_NIL && - (e_hash & prefix_mask) == current)) { - struct hash_subtable *new_st = hash_subtable_new (h, 1, HASH_USED (new_flags)); - int32 rc = hash_insert_internal (t, &new_st, new_flags, e->hash, h, e->data, &dummy_result); - assert (rc == 0); - memcpy(dummy_result, e->data, h->datasize); - e = ne; - while (e <= st->last && (e_hash = e->hash) != HASH_NIL && (e_hash & prefix_mask) == current) { - assert ((e_hash & HASH_MASK) != HASH_SUBHASH); - rc = hash_insert_internal (t, &new_st, new_flags, e_hash, h, e->data, &dummy_result); - assert (rc == 0); - memcpy(dummy_result, e->data, h->datasize); - e = HASH_OFFSET (e, elemsize); - } - memset (de->data, HASH_NIL_MEMSET, h->datasize); - *(struct hash_subtable **)de->data = new_st; - de->hash = current | HASH_SUBHASH; - } else { - if (e != de) { - memcpy (de, e, elemsize); - } - e = HASH_OFFSET (e, elemsize); - } - de = HASH_OFFSET (de, elemsize); - } - if (e != de) { - hash_remove_n (st, de, (((byte *) e) - (byte *) de) / elemsize); - } + noldbuckets = (uintptr)1 << (h->B - 1); + + // make sure we evacuate the oldbucket corresponding + // to the bucket we're about to use + evacuate(t, h, bucket & (noldbuckets - 1)); + + // evacuate one more oldbucket to make progress on growing + if(h->oldbuckets != nil) + evacuate(t, h, h->nevacuate); } static void -hash_grow (MapType *t, Hmap *h, struct hash_subtable **pst, int32 flags) +hash_grow(MapType *t, Hmap *h) { - struct hash_subtable *old_st = *pst; - int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - *pst = hash_subtable_new (h, old_st->power + 1, HASH_USED (flags)); - struct hash_entry *last_e = old_st->last; - struct hash_entry *e; - void *dummy_result; - int32 used = 0; - - flags |= HASH_REHASH; - for (e = old_st->entry; e <= last_e; e = HASH_OFFSET (e, elemsize)) { - hash_hash_t hash = e->hash; - if (hash != HASH_NIL) { - int32 rc = hash_insert_internal (t, pst, flags, e->hash, h, e->data, &dummy_result); - assert (rc == 0); - memcpy(dummy_result, e->data, h->datasize); - used++; - } + byte *old_buckets; + byte *new_buckets; + uint8 flags; + + // allocate a bigger hash table + if(h->oldbuckets != nil) + runtime·throw("evacuation not done in time"); + old_buckets = h->buckets; + // NOTE: this could be a big malloc, but since we don't need zeroing it is probably fast. + new_buckets = runtime·mallocgc(h->bucketsize << (h->B + 1), 0, 1, 0); + flags = (h->flags & ~(Iterator | OldIterator)); + if((h->flags & Iterator) != 0) { + flags |= OldIterator; + // We can't free indirect keys any more, as + // they are potentially aliased across buckets. + flags &= ~CanFreeKey; } - if (h->flag & CanFreeTable) - free (old_st); + + // commit the grow (atomic wrt gc) + h->B++; + h->flags = flags; + h->oldbuckets = old_buckets; + h->buckets = new_buckets; + h->nevacuate = 0; + + // the actual copying of the hash table data is done incrementally + // by grow_work() and evacuate(). + if(docheck) + check(t, h); } -static int32 -hash_lookup (MapType *t, Hmap *h, void *data, void **pres) +// returns ptr to value associated with key *keyp, or nil if none. +// if it returns non-nil, updates *keyp to point to the currently stored key. +static byte* +hash_lookup(MapType *t, Hmap *h, byte **keyp) { - int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - hash_hash_t hash; - struct hash_subtable *st = h->st; - int32 used = 0; - hash_hash_t e_hash; - struct hash_entry *e; - struct hash_entry *end_e; void *key; + uintptr hash; + uintptr bucket; + Bucket *b; + uint8 top; + uintptr i; bool eq; + byte *k, *k2, *v; + key = *keyp; + if(docheck) + check(t, h); hash = h->hash0; - (*t->key->alg->hash) (&hash, t->key->size, data); - hash &= ~HASH_MASK; - hash += HASH_ADJUST (hash); - for (;;) { - int32 shift = HASH_BITS - (st->power + used); - int32 index_mask = (1 << st->power) - 1; - int32 i = (hash >> shift) & index_mask; /* i is the natural position of hash */ - - e = HASH_OFFSET (st->entry, i * elemsize); /* e points to element i */ - e_hash = e->hash; - if ((e_hash & HASH_MASK) != HASH_SUBHASH) { /* a subtable */ - break; - } - used += st->power; - st = *(struct hash_subtable **)e->data; - } - end_e = HASH_OFFSET (e, st->limit_bytes); - while (e != end_e && (e_hash = e->hash) != HASH_NIL && e_hash < hash) { - e = HASH_OFFSET (e, elemsize); - } - while (e != end_e && ((e_hash = e->hash) ^ hash) < HASH_SUBHASH) { - key = e->data; - if (h->flag & IndirectKey) - key = *(void**)e->data; - if (HASH_DATA_EQ (eq, t, h, data, key)) { /* a match */ - *pres = e->data; - return (1); + t->key->alg->hash(&hash, t->key->size, key); + bucket = hash & (((uintptr)1 << h->B) - 1); + if(h->oldbuckets != nil) + grow_work(t, h, bucket); + b = (Bucket*)(h->buckets + bucket * h->bucketsize); + top = hash >> (sizeof(uintptr)*8 - 8); + if(top == 0) + top = 1; + do { + for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] == top) { + k2 = IK(h, k); + t->key->alg->equal(&eq, t->key->size, key, k2); + if(eq) { + *keyp = k2; + return IV(h, v); + } + } } - e = HASH_OFFSET (e, elemsize); - } - USED(e_hash); - *pres = 0; - return (0); + b = b->overflow; + } while(b != nil); + return nil; } -static int32 -hash_remove (MapType *t, Hmap *h, void *data) +// When an item is not found, fast versions return a pointer to this zeroed memory. +static uint8 empty_value[MAXVALUESIZE]; + +// Specialized versions of mapaccess1 for specific types. +// See ./hashmap_fast and ../../cmd/gc/walk.c. +#define HASH_LOOKUP1 runtime·mapaccess1_fast32 +#define HASH_LOOKUP2 runtime·mapaccess2_fast32 +#define KEYTYPE uint32 +#define HASHFUNC runtime·algarray[AMEM32].hash +#define EQFUNC(x,y) ((x) == (y)) +#define QUICKEQ(x) true +#include "hashmap_fast.c" + +#undef HASH_LOOKUP1 +#undef HASH_LOOKUP2 +#undef KEYTYPE +#undef HASHFUNC +#undef EQFUNC +#undef QUICKEQ + +#define HASH_LOOKUP1 runtime·mapaccess1_fast64 +#define HASH_LOOKUP2 runtime·mapaccess2_fast64 +#define KEYTYPE uint64 +#define HASHFUNC runtime·algarray[AMEM64].hash +#define EQFUNC(x,y) ((x) == (y)) +#define QUICKEQ(x) true +#include "hashmap_fast.c" + +#undef HASH_LOOKUP1 +#undef HASH_LOOKUP2 +#undef KEYTYPE +#undef HASHFUNC +#undef EQFUNC +#undef QUICKEQ + +#define HASH_LOOKUP1 runtime·mapaccess1_faststr +#define HASH_LOOKUP2 runtime·mapaccess2_faststr +#define KEYTYPE String +#define HASHFUNC runtime·algarray[ASTRING].hash +#define EQFUNC(x,y) ((x).len == (y).len && ((x).str == (y).str || runtime·mcmp((x).str, (y).str, (x).len) == 0)) +#define QUICKEQ(x) ((x).len < 32) +#include "hashmap_fast.c" + +static void +hash_insert(MapType *t, Hmap *h, void *key, void *value) { - int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - hash_hash_t hash; - struct hash_subtable *st = h->st; - int32 used = 0; - hash_hash_t e_hash; - struct hash_entry *e; - struct hash_entry *end_e; + uintptr hash; + uintptr bucket; + uintptr i; bool eq; - void *key; - + Bucket *b; + Bucket *newb; + uint8 *inserti; + byte *insertk, *insertv; + uint8 top; + byte *k, *v; + byte *kmem, *vmem; + + if(docheck) + check(t, h); hash = h->hash0; - (*t->key->alg->hash) (&hash, t->key->size, data); - hash &= ~HASH_MASK; - hash += HASH_ADJUST (hash); - for (;;) { - int32 shift = HASH_BITS - (st->power + used); - int32 index_mask = (1 << st->power) - 1; - int32 i = (hash >> shift) & index_mask; /* i is the natural position of hash */ - - e = HASH_OFFSET (st->entry, i * elemsize); /* e points to element i */ - e_hash = e->hash; - if ((e_hash & HASH_MASK) != HASH_SUBHASH) { /* a subtable */ - break; + t->key->alg->hash(&hash, t->key->size, key); + again: + bucket = hash & (((uintptr)1 << h->B) - 1); + if(h->oldbuckets != nil) + grow_work(t, h, bucket); + b = (Bucket*)(h->buckets + bucket * h->bucketsize); + top = hash >> (sizeof(uintptr)*8 - 8); + if(top == 0) + top = 1; + inserti = 0; + insertk = nil; + insertv = nil; + while(true) { + for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] != top) { + if(b->tophash[i] == 0 && inserti == nil) { + inserti = &b->tophash[i]; + insertk = k; + insertv = v; + } + continue; + } + t->key->alg->equal(&eq, t->key->size, key, IK(h, k)); + if(!eq) + continue; + // already have a mapping for key. Update it. + t->key->alg->copy(t->key->size, IK(h, k), key); // Need to update key for keys which are distinct but equal (e.g. +0.0 and -0.0) + t->elem->alg->copy(t->elem->size, IV(h, v), value); + if(docheck) + check(t, h); + return; } - used += st->power; - st = *(struct hash_subtable **)e->data; - } - end_e = HASH_OFFSET (e, st->limit_bytes); - while (e != end_e && (e_hash = e->hash) != HASH_NIL && e_hash < hash) { - e = HASH_OFFSET (e, elemsize); + if(b->overflow == nil) + break; + b = b->overflow; } - while (e != end_e && ((e_hash = e->hash) ^ hash) < HASH_SUBHASH) { - key = e->data; - if (h->flag & IndirectKey) - key = *(void**)e->data; - if (HASH_DATA_EQ (eq, t, h, data, key)) { /* a match */ - // Free key if indirect, but only if reflect can't be - // holding a pointer to it. Deletions are rare, - // indirect (large) keys are rare, reflect on maps - // is rare. So in the rare, rare, rare case of deleting - // an indirect key from a map that has been reflected on, - // we leave the key for garbage collection instead of - // freeing it here. - if (h->flag & CanFreeKey) - free (key); - if (h->flag & IndirectVal) - free (*(void**)((byte*)e->data + h->valoff)); - hash_remove_n (st, e, 1); - h->count--; - return (1); - } - e = HASH_OFFSET (e, elemsize); + + // did not find mapping for key. Allocate new cell & add entry. + if(h->count >= LOAD * ((uintptr)1 << h->B) && h->count >= BUCKETSIZE) { + hash_grow(t, h); + goto again; // Growing the table invalidates everything, so try again } - USED(e_hash); - return (0); -} -static int32 -hash_insert_internal (MapType *t, struct hash_subtable **pst, int32 flags, hash_hash_t hash, - Hmap *h, void *data, void **pres) -{ - int32 elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - bool eq; + if(inserti == nil) { + // all current buckets are full, allocate a new one. + newb = runtime·mallocgc(h->bucketsize, 0, 1, 0); + clearbucket(newb); + b->overflow = newb; + inserti = newb->tophash; + insertk = newb->data; + insertv = insertk + h->keysize * BUCKETSIZE; + } - if ((flags & HASH_REHASH) == 0) { - hash += HASH_ADJUST (hash); - hash &= ~HASH_MASK; + // store new key/value at insert position + if((h->flags & IndirectKey) != 0) { + kmem = runtime·mallocgc(t->key->size, 0, 1, 0); + *(byte**)insertk = kmem; + insertk = kmem; } - for (;;) { - struct hash_subtable *st = *pst; - int32 shift = HASH_BITS - (st->power + HASH_USED (flags)); - int32 index_mask = (1 << st->power) - 1; - int32 i = (hash >> shift) & index_mask; /* i is the natural position of hash */ - struct hash_entry *start_e = - HASH_OFFSET (st->entry, i * elemsize); /* start_e is the pointer to element i */ - struct hash_entry *e = start_e; /* e is going to range over [start_e, end_e) */ - struct hash_entry *end_e; - hash_hash_t e_hash = e->hash; - - if ((e_hash & HASH_MASK) == HASH_SUBHASH) { /* a subtable */ - pst = (struct hash_subtable **) e->data; - flags += HASH_MAKE_USED (st->power); - continue; - } - end_e = HASH_OFFSET (start_e, st->limit_bytes); - while (e != end_e && (e_hash = e->hash) != HASH_NIL && e_hash < hash) { - e = HASH_OFFSET (e, elemsize); - i++; - } - if (e != end_e && e_hash != HASH_NIL) { - /* ins_e ranges over the elements that may match */ - struct hash_entry *ins_e = e; - int32 ins_i = i; - hash_hash_t ins_e_hash; - void *key; - while (ins_e != end_e && ((e_hash = ins_e->hash) ^ hash) < HASH_SUBHASH) { - key = ins_e->data; - if (h->flag & IndirectKey) - key = *(void**)key; - if (HASH_DATA_EQ (eq, t, h, data, key)) { /* a match */ - *pres = ins_e->data; - return (1); - } - if (e_hash == hash) { /* adjust hash if it collides */ - assert ((flags & HASH_REHASH) == 0); - hash++; - if ((hash & HASH_MASK) == HASH_SUBHASH) - runtime·throw("runtime: map hash collision overflow"); - } - ins_e = HASH_OFFSET (ins_e, elemsize); - ins_i++; - if (e_hash <= hash) { /* set e to insertion point */ - e = ins_e; - i = ins_i; - } - } - /* set ins_e to the insertion point for the new element */ - ins_e = e; - ins_i = i; - ins_e_hash = 0; - /* move ins_e to point at the end of the contiguous block, but - stop if any element can't be moved by one up */ - while (ins_e <= st->last && (ins_e_hash = ins_e->hash) != HASH_NIL && - ins_i + 1 - ((ins_e_hash >> shift) & index_mask) < st->max_probes && - (ins_e_hash & HASH_MASK) != HASH_SUBHASH) { - ins_e = HASH_OFFSET (ins_e, elemsize); - ins_i++; - } - if (e == end_e || ins_e > st->last || ins_e_hash != HASH_NIL) { - e = end_e; /* can't insert; must grow or convert to subtable */ - } else { /* make space for element */ - memmove (HASH_OFFSET (e, elemsize), e, ((byte *) ins_e) - (byte *) e); - } - } - if (e != end_e) { - e->hash = hash; - *pres = e->data; - return (0); - } - h->changes++; - if (st->power < HASH_MAX_POWER) { - hash_grow (t, h, pst, flags); - } else { - hash_conv (t, h, st, flags, hash, start_e); - } + if((h->flags & IndirectValue) != 0) { + vmem = runtime·mallocgc(t->elem->size, 0, 1, 0); + *(byte**)insertv = vmem; + insertv = vmem; } + t->key->alg->copy(t->key->size, insertk, key); + t->elem->alg->copy(t->elem->size, insertv, value); + *inserti = top; + h->count++; + if(docheck) + check(t, h); } -static int32 -hash_insert (MapType *t, Hmap *h, void *data, void **pres) +static void +hash_remove(MapType *t, Hmap *h, void *key) { uintptr hash; - int32 rc; - + uintptr bucket; + Bucket *b; + uint8 top; + uintptr i; + byte *k, *v; + bool eq; + + if(docheck) + check(t, h); hash = h->hash0; - (*t->key->alg->hash) (&hash, t->key->size, data); - rc = hash_insert_internal (t, &h->st, 0, hash, h, data, pres); + t->key->alg->hash(&hash, t->key->size, key); + bucket = hash & (((uintptr)1 << h->B) - 1); + if(h->oldbuckets != nil) + grow_work(t, h, bucket); + b = (Bucket*)(h->buckets + bucket * h->bucketsize); + top = hash >> (sizeof(uintptr)*8 - 8); + if(top == 0) + top = 1; + do { + for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] != top) + continue; + t->key->alg->equal(&eq, t->key->size, key, IK(h, k)); + if(!eq) + continue; + + if((h->flags & CanFreeKey) != 0) { + k = *(byte**)k; + } + if((h->flags & IndirectValue) != 0) { + v = *(byte**)v; + } - h->count += (rc == 0); /* increment count if element didn't previously exist */ - return (rc); + b->tophash[i] = 0; + h->count--; + + if((h->flags & CanFreeKey) != 0) { + runtime·free(k); + } + if((h->flags & IndirectValue) != 0) { + runtime·free(v); + } + // TODO: consolidate buckets if they are mostly empty + // can only consolidate if there are no live iterators at this size. + if(docheck) + check(t, h); + return; + } + b = b->overflow; + } while(b != nil); } -static uint32 -hash_count (Hmap *h) -{ - return (h->count); -} +// TODO: shrink the map, the same way we grow it. -static void -iter_restart (struct hash_iter *it, struct hash_subtable *st, int32 used) +// If you modify hash_iter, also change cmd/gc/range.c to indicate +// the size of this structure. +struct hash_iter { - int32 elemsize = it->elemsize; - hash_hash_t last_hash = it->last_hash; - struct hash_entry *e; - hash_hash_t e_hash; - struct hash_iter_sub *sub = &it->subtable_state[it->i]; - struct hash_entry *last; - - for (;;) { - int32 shift = HASH_BITS - (st->power + used); - int32 index_mask = (1 << st->power) - 1; - int32 i = (last_hash >> shift) & index_mask; - - last = st->last; - e = HASH_OFFSET (st->entry, i * elemsize); - sub->start = st->entry; - sub->last = last; - - if ((e->hash & HASH_MASK) != HASH_SUBHASH) { - break; - } - sub->e = HASH_OFFSET (e, elemsize); - sub = &it->subtable_state[++(it->i)]; - used += st->power; - st = *(struct hash_subtable **)e->data; - } - while (e <= last && ((e_hash = e->hash) == HASH_NIL || e_hash <= last_hash)) { - e = HASH_OFFSET (e, elemsize); - } - sub->e = e; -} + uint8* key; // Must be in first position. Write nil to indicate iteration end (see cmd/gc/range.c). + uint8* value; -static void * -hash_next (struct hash_iter *it) -{ - int32 elemsize; - struct hash_iter_sub *sub; - struct hash_entry *e; - struct hash_entry *last; - hash_hash_t e_hash; - - if (it->changes != it->h->changes) { /* hash table's structure changed; recompute */ - if (~it->last_hash == 0) - return (0); - it->changes = it->h->changes; - it->i = 0; - iter_restart (it, it->h->st, 0); - } - elemsize = it->elemsize; - -Again: - e_hash = 0; - sub = &it->subtable_state[it->i]; - e = sub->e; - last = sub->last; - - if (e != sub->start && it->last_hash != HASH_OFFSET (e, -elemsize)->hash) { - struct hash_entry *start = HASH_OFFSET (e, -(elemsize * HASH_MAX_PROBES)); - struct hash_entry *pe = HASH_OFFSET (e, -elemsize); - hash_hash_t last_hash = it->last_hash; - if (start < sub->start) { - start = sub->start; - } - while (e != start && ((e_hash = pe->hash) == HASH_NIL || last_hash < e_hash)) { - e = pe; - pe = HASH_OFFSET (pe, -elemsize); - } - while (e <= last && ((e_hash = e->hash) == HASH_NIL || e_hash <= last_hash)) { - e = HASH_OFFSET (e, elemsize); - } - } + MapType *t; + Hmap *h; - for (;;) { - while (e <= last && (e_hash = e->hash) == HASH_NIL) { - e = HASH_OFFSET (e, elemsize); - } - if (e > last) { - if (it->i == 0) { - if(!it->cycled) { - // Wrap to zero and iterate up until it->cycle. - it->cycled = true; - it->last_hash = 0; - it->subtable_state[0].e = it->h->st->entry; - it->subtable_state[0].start = it->h->st->entry; - it->subtable_state[0].last = it->h->st->last; - goto Again; - } - // Set last_hash to impossible value and - // break it->changes, so that check at top of - // hash_next will be used if we get called again. - it->last_hash = ~(uintptr_t)0; - it->changes--; - return (0); - } else { - it->i--; - sub = &it->subtable_state[it->i]; - e = sub->e; - last = sub->last; - } - } else if ((e_hash & HASH_MASK) != HASH_SUBHASH) { - if(it->cycled && e->hash > it->cycle) { - // Already returned this. - // Set last_hash to impossible value and - // break it->changes, so that check at top of - // hash_next will be used if we get called again. - it->last_hash = ~(uintptr_t)0; - it->changes--; - return (0); - } - it->last_hash = e->hash; - sub->e = HASH_OFFSET (e, elemsize); - return (e->data); - } else { - struct hash_subtable *st = - *(struct hash_subtable **)e->data; - sub->e = HASH_OFFSET (e, elemsize); - it->i++; - assert (it->i < sizeof (it->subtable_state) / - sizeof (it->subtable_state[0])); - sub = &it->subtable_state[it->i]; - sub->e = e = st->entry; - sub->start = st->entry; - sub->last = last = st->last; - } - } -} + // end point for iteration + uintptr endbucket; + bool wrapped; + // state of table at time iterator is initialized + uint8 B; + byte *buckets; + + // iter state + uintptr bucket; + struct Bucket *bptr; + uintptr i; +}; + +// iterator state: +// bucket: the current bucket ID +// b: the current Bucket in the chain +// i: the next offset to check in the current bucket static void -hash_iter_init (MapType *t, Hmap *h, struct hash_iter *it) +hash_iter_init(MapType *t, Hmap *h, struct hash_iter *it) { - it->elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - it->changes = h->changes; - it->i = 0; - it->h = h; it->t = t; - it->last_hash = 0; - it->subtable_state[0].e = h->st->entry; - it->subtable_state[0].start = h->st->entry; - it->subtable_state[0].last = h->st->last; - - // fastrand1 returns 31 useful bits. - // We don't care about not having a bottom bit but we - // do want top bits. - if(sizeof(void*) == 8) - it->cycle = (uint64)runtime·fastrand1()<<33 | (uint64)runtime·fastrand1()<<2; - else - it->cycle = runtime·fastrand1()<<1; - it->cycled = false; - it->last_hash = it->cycle; - iter_restart(it, it->h->st, 0); -} + it->h = h; -static void -clean_st (Hmap *h, struct hash_subtable *st, int32 *slots, int32 *used) -{ - int32 elemsize = st->datasize + offsetof (struct hash_entry, data[0]); - struct hash_entry *e = st->entry; - struct hash_entry *last = st->last; - int32 lslots = (((byte *) (last+1)) - (byte *) e) / elemsize; - int32 lused = 0; - - while (e <= last) { - hash_hash_t hash = e->hash; - if ((hash & HASH_MASK) == HASH_SUBHASH) { - clean_st (h, *(struct hash_subtable **)e->data, slots, used); - } else { - lused += (hash != HASH_NIL); - } - e = HASH_OFFSET (e, elemsize); - } - if (h->flag & CanFreeTable) - free (st); - *slots += lslots; - *used += lused; -} + // grab snapshot of bucket state + it->B = h->B; + it->buckets = h->buckets; -static void -hash_destroy (Hmap *h) -{ - int32 slots = 0; - int32 used = 0; + // iterator state + it->bucket = it->endbucket = runtime·fastrand1() & (((uintptr)1 << h->B) - 1); + it->wrapped = false; + it->bptr = nil; - clean_st (h, h->st, &slots, &used); - free (h); + // Remember we have an iterator at this level. + h->flags |= Iterator; } +// initializes it->key and it->value to the next key/value pair +// in the iteration, or nil if we've reached the end. static void -hash_visit_internal (struct hash_subtable *st, - int32 used, int32 level, - void (*data_visit) (void *arg, int32 level, void *data), - void *arg) +hash_next(struct hash_iter *it) { - int32 elemsize = st->datasize + offsetof (struct hash_entry, data[0]); - struct hash_entry *e = st->entry; - int32 shift = HASH_BITS - (used + st->power); - int32 i = 0; - - while (e <= st->last) { - int32 index = ((e->hash >> (shift - 1)) >> 1) & ((1 << st->power) - 1); - if ((e->hash & HASH_MASK) == HASH_SUBHASH) { - (*data_visit) (arg, level, e->data); - hash_visit_internal (*(struct hash_subtable **)e->data, - used + st->power, level + 1, data_visit, arg); - } else { - (*data_visit) (arg, level, e->data); + Hmap *h; + MapType *t; + uintptr bucket; + Bucket *b; + uintptr i; + bool eq; + byte *k, *v; + byte *rk, *rv; + + h = it->h; + t = it->t; + bucket = it->bucket; + b = it->bptr; + i = it->i; + +next: + if(b == nil) { + if(bucket == it->endbucket && it->wrapped) { + // end of iteration + it->key = nil; + it->value = nil; + return; + } + if(h->oldbuckets != nil && it->B == h->B) { + // Iterator was started in the middle of a grow, and the grow isn't done yet. + // Make sure the bucket we're about to read is valid. + grow_work(t, h, bucket); } - if (e->hash != HASH_NIL) { - assert (i < index + st->max_probes); - assert (index <= i); + b = (Bucket*)(it->buckets + bucket * h->bucketsize); + bucket++; + if(bucket == ((uintptr)1 << it->B)) { + bucket = 0; + it->wrapped = true; + } + i = 0; + } + k = b->data + h->keysize * i; + v = b->data + h->keysize * BUCKETSIZE + h->valuesize * i; + for(; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] != 0) { + if(!evacuated(b)) { + // this is the golden data, we can return it. + it->key = IK(h, k); + it->value = IV(h, v); + } else { + // The hash table has grown since the iterator was started. + // The golden data for this key is now somewhere else. + t->key->alg->equal(&eq, t->key->size, IK(h, k), IK(h, k)); + if(eq) { + // Check the current hash table for the data. + // This code handles the case where the key + // has been deleted, updated, or deleted and reinserted. + // NOTE: we need to regrab the key as it has potentially been + // updated to an equal() but not identical key (e.g. +0.0 vs -0.0). + rk = IK(h, k); + rv = hash_lookup(t, it->h, &rk); + if(rv == nil) + continue; // key has been deleted + it->key = rk; + it->value = rv; + } else { + // if key!=key then the entry can't be deleted or + // updated, so we can just return it. That's lucky for + // us because when key!=key we can't look it up + // successfully in the current table. + it->key = IK(h, k); + it->value = IV(h, v); + } + } + it->bucket = bucket; + it->bptr = b; + it->i = i + 1; + return; } - e = HASH_OFFSET (e, elemsize); - i++; } + b = overflowptr(b); + i = 0; + goto next; } -static void -hash_visit (Hmap *h, void (*data_visit) (void *arg, int32 level, void *data), void *arg) -{ - hash_visit_internal (h->st, 0, 0, data_visit, arg); -} + +#define PHASE_BUCKETS 0 +#define PHASE_OLD_BUCKETS 1 +#define PHASE_TABLE 2 +#define PHASE_OLD_TABLE 3 +#define PHASE_DONE 4 // Initialize the iterator. // Returns false if Hmap contains no pointers (in which case the iterator is not initialized). @@ -713,73 +841,141 @@ bool hash_gciter_init (Hmap *h, struct hash_gciter *it) { // GC during map initialization - if(h->st == nil) + if(h->buckets == nil) return false; - it->elemsize = h->datasize + offsetof (struct hash_entry, data[0]); - it->flag = h->flag; - it->valoff = h->valoff; - it->i = 0; - it->st = h->st; - it->subtable_state[it->i].e = h->st->entry; - it->subtable_state[it->i].last = h->st->last; + it->h = h; + it->phase = PHASE_BUCKETS; + it->bucket = 0; + it->b = nil; + + // TODO: finish evacuating oldbuckets so that we can collect + // oldbuckets? We don't want to keep a partially evacuated + // table around forever, so each gc could make at least some + // evacuation progress. Need to be careful about concurrent + // access if we do concurrent gc. Even if not, we don't want + // to make the gc pause any longer than it has to be. + return true; } -// Returns true and fills *data with subtable/key/value data, +// Returns true and fills *data with internal structure/key/value data, // or returns false if the iterator has terminated. +// Ugh, this interface is really annoying. I want a callback fn! bool -hash_gciter_next (struct hash_gciter *it, struct hash_gciter_data *data) +hash_gciter_next(struct hash_gciter *it, struct hash_gciter_data *data) { - struct hash_entry *e; - struct hash_gciter_sub *sub; + Hmap *h; + uintptr bucket, oldbucket; + Bucket *b, *oldb; + uintptr i; + byte *k, *v; + + h = it->h; + bucket = it->bucket; + b = it->b; + i = it->i; data->st = nil; data->key_data = nil; data->val_data = nil; - - // pointer to the first-level table - if(it->st != nil) { - data->st = it->st; - it->st = nil; - return true; - } - -popped: - sub = &it->subtable_state[it->i]; - e = sub->e; - while (e <= sub->last) { - if ((e->hash & HASH_MASK) == HASH_SUBHASH) { - struct hash_subtable *st = *(struct hash_subtable **)e->data; - data->st = st; - sub->e = HASH_OFFSET (e, it->elemsize); - - // push - it->i++; - assert (it->i < nelem(it->subtable_state)); - sub++; - sub->e = st->entry; - sub->last = st->last; - - return true; + data->indirectkey = (h->flags & IndirectKey) != 0; + data->indirectval = (h->flags & IndirectValue) != 0; + +next: + switch (it->phase) { + case PHASE_BUCKETS: + if(b != nil) { + k = b->data + h->keysize * i; + v = b->data + h->keysize * BUCKETSIZE + h->valuesize * i; + for(; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] != 0) { + data->key_data = k; + data->val_data = v; + it->bucket = bucket; + it->b = b; + it->i = i + 1; + return true; + } + } + b = b->overflow; + if(b != nil) { + data->st = (byte*)b; + it->bucket = bucket; + it->b = b; + it->i = 0; + return true; + } + } + while(bucket < ((uintptr)1 << h->B)) { + if(h->oldbuckets != nil) { + oldbucket = bucket & (((uintptr)1 << (h->B - 1)) - 1); + oldb = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize); + if(!evacuated(oldb)) { + // new bucket isn't valid yet + bucket++; + continue; + } + } + b = (Bucket*)(h->buckets + bucket * h->bucketsize); + i = 0; + bucket++; + goto next; + } + it->phase = PHASE_OLD_BUCKETS; + bucket = 0; + b = nil; + goto next; + case PHASE_OLD_BUCKETS: + if(h->oldbuckets == nil) { + it->phase = PHASE_TABLE; + goto next; + } + if(b != nil) { + k = b->data + h->keysize * i; + v = b->data + h->keysize * BUCKETSIZE + h->valuesize * i; + for(; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) { + if(b->tophash[i] != 0) { + data->key_data = k; + data->val_data = v; + it->bucket = bucket; + it->b = b; + it->i = i + 1; + return true; + } + } + b = overflowptr(b); + if(b != nil) { + data->st = (byte*)b; + it->bucket = bucket; + it->b = b; + it->i = 0; + return true; + } + } + if(bucket < ((uintptr)1 << (h->B - 1))) { + b = (Bucket*)(h->oldbuckets + bucket * h->bucketsize); + bucket++; + i = 0; + goto next; } - if(e->hash != HASH_NIL) { - void *key_data = e->data; - void *val_data = (byte*)e->data + it->valoff; - data->key_data = key_data; - data->val_data = val_data; - data->indirectkey = (it->flag & IndirectKey) != 0; - data->indirectval = (it->flag & IndirectVal) != 0; - sub->e = HASH_OFFSET (e, it->elemsize); + it->phase = PHASE_TABLE; + goto next; + case PHASE_TABLE: + it->phase = PHASE_OLD_TABLE; + data->st = h->buckets; + return true; + case PHASE_OLD_TABLE: + it->phase = PHASE_DONE; + if(h->oldbuckets != nil) { + data->st = h->oldbuckets; return true; + } else { + goto next; } - e = HASH_OFFSET (e, it->elemsize); - } - if(it->i != 0) { - // pop - it->i--; - goto popped; } + if(it->phase != PHASE_DONE) + runtime·throw("bad phase at done"); return false; } @@ -787,35 +983,13 @@ popped: /// interfaces to go runtime // -static void** -hash_valptr(Hmap *h, void *p) -{ - p = (byte*)p + h->valoff; - if(h->flag & IndirectVal) - p = *(void**)p; - return p; -} - - -static void** -hash_keyptr(Hmap *h, void *p) -{ - if(h->flag & IndirectKey) - p = *(void**)p; - return p; -} - -static int32 debug = 0; - Hmap* runtime·makemap_c(MapType *typ, int64 hint) { Hmap *h; - Type *key, *val; - uintptr ksize, vsize; + Type *key; key = typ->key; - val = typ->elem; if(hint < 0 || (int32)hint != hint) runtime·panicstring("makemap: size out of range"); @@ -824,7 +998,6 @@ runtime·makemap_c(MapType *typ, int64 hint) runtime·throw("runtime.makemap: unsupported map key type"); h = runtime·mal(sizeof(*h)); - h->flag |= CanFreeTable; /* until reflect gets involved, free is okay */ if(UseSpanType) { if(false) { @@ -833,34 +1006,14 @@ runtime·makemap_c(MapType *typ, int64 hint) runtime·settype(h, (uintptr)typ | TypeInfo_Map); } - ksize = ROUND(key->size, sizeof(void*)); - vsize = ROUND(val->size, sizeof(void*)); - if(ksize > MaxData || vsize > MaxData || ksize+vsize > MaxData) { - // Either key is too big, or value is, or combined they are. - // Prefer to keep the key if possible, because we look at - // keys more often than values. - if(ksize > MaxData - sizeof(void*)) { - // No choice but to indirect the key. - h->flag |= IndirectKey; - h->flag |= CanFreeKey; /* until reflect gets involved, free is okay */ - ksize = sizeof(void*); - } - if(vsize > MaxData - ksize) { - // Have to indirect the value. - h->flag |= IndirectVal; - vsize = sizeof(void*); - } - } - - h->valoff = ksize; - hash_init(h, ksize+vsize, hint); + hash_init(typ, h, hint); // these calculations are compiler dependent. // figure out offsets of map call arguments. if(debug) { runtime·printf("makemap: map=%p; keysize=%p; valsize=%p; keyalg=%p; valalg=%p\n", - h, key->size, val->size, key->alg, val->alg); + h, key->size, typ->elem->size, key->alg, typ->elem->alg); } return h; @@ -890,7 +1043,7 @@ runtime·mapaccess(MapType *t, Hmap *h, byte *ak, byte *av, bool *pres) Type *elem; elem = t->elem; - if(h == nil) { + if(h == nil || h->count == 0) { elem->alg->copy(elem->size, av, nil); *pres = false; return; @@ -899,10 +1052,11 @@ runtime·mapaccess(MapType *t, Hmap *h, byte *ak, byte *av, bool *pres) if(runtime·gcwaiting) runtime·gosched(); - res = nil; - if(hash_lookup(t, h, ak, (void**)&res)) { + res = hash_lookup(t, h, &ak); + + if(res != nil) { *pres = true; - elem->alg->copy(elem->size, av, hash_valptr(h, res)); + elem->alg->copy(elem->size, av, res); } else { *pres = false; elem->alg->copy(elem->size, av, nil); @@ -915,7 +1069,7 @@ void runtime·mapaccess1(MapType *t, Hmap *h, ...) { byte *ak, *av; - bool pres; + byte *res; if(raceenabled && h != nil) runtime·racereadpc(h, runtime·getcallerpc(&t), runtime·mapaccess1); @@ -923,7 +1077,12 @@ runtime·mapaccess1(MapType *t, Hmap *h, ...) ak = (byte*)(&h + 1); av = ak + ROUND(t->key->size, Structrnd); - runtime·mapaccess(t, h, ak, av, &pres); + if(h == nil || h->count == 0) { + t->elem->alg->copy(t->elem->size, av, nil); + } else { + res = hash_lookup(t, h, &ak); + t->elem->alg->copy(t->elem->size, av, res); + } if(debug) { runtime·prints("runtime.mapaccess1: map="); @@ -932,8 +1091,6 @@ runtime·mapaccess1(MapType *t, Hmap *h, ...) t->key->alg->print(t->key->size, ak); runtime·prints("; val="); t->elem->alg->print(t->elem->size, av); - runtime·prints("; pres="); - runtime·printbool(pres); runtime·prints("\n"); } } @@ -960,7 +1117,7 @@ runtime·mapaccess2(MapType *t, Hmap *h, ...) runtime·prints("; key="); t->key->alg->print(t->key->size, ak); runtime·prints("; val="); - t->elem->alg->print(t->key->size, av); + t->elem->alg->print(t->elem->size, av); runtime·prints("; pres="); runtime·printbool(*ap); runtime·prints("\n"); @@ -999,9 +1156,6 @@ reflect·mapaccess(MapType *t, Hmap *h, uintptr key, uintptr val, bool pres) void runtime·mapassign(MapType *t, Hmap *h, byte *ak, byte *av) { - byte *res; - int32 hit; - if(h == nil) runtime·panicstring("assignment to entry in nil map"); @@ -1010,19 +1164,9 @@ runtime·mapassign(MapType *t, Hmap *h, byte *ak, byte *av) if(av == nil) { hash_remove(t, h, ak); - return; - } - - res = nil; - hit = hash_insert(t, h, ak, (void**)&res); - if(!hit) { - if(h->flag & IndirectKey) - *(void**)res = runtime·mal(t->key->size); - if(h->flag & IndirectVal) - *(void**)(res+h->valoff) = runtime·mal(t->elem->size); + } else { + hash_insert(t, h, ak, av); } - t->key->alg->copy(t->key->size, hash_keyptr(h, res), ak); - t->elem->alg->copy(t->elem->size, hash_valptr(h, res), av); if(debug) { runtime·prints("mapassign: map="); @@ -1031,10 +1175,6 @@ runtime·mapassign(MapType *t, Hmap *h, byte *ak, byte *av) t->key->alg->print(t->key->size, ak); runtime·prints("; val="); t->elem->alg->print(t->elem->size, av); - runtime·prints("; hit="); - runtime·printint(hit); - runtime·prints("; res="); - runtime·printpointer(res); runtime·prints("\n"); } } @@ -1112,20 +1252,20 @@ void runtime·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it) { if(h == nil) { - it->data = nil; + it->key = nil; return; } if(raceenabled) runtime·racereadpc(h, runtime·getcallerpc(&t), runtime·mapiterinit); hash_iter_init(t, h, it); - it->data = hash_next(it); + hash_next(it); if(debug) { runtime·prints("runtime.mapiterinit: map="); runtime·printpointer(h); runtime·prints("; iter="); runtime·printpointer(it); - runtime·prints("; data="); - runtime·printpointer(it->data); + runtime·prints("; key="); + runtime·printpointer(it->key); runtime·prints("\n"); } } @@ -1135,20 +1275,18 @@ runtime·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it) void reflect·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it) { - uint8 flag; + uint8 flags; if(h != nil && t->key->size > sizeof(void*)) { // reflect·mapiterkey returns pointers to key data, // and reflect holds them, so we cannot free key data - // eagerly anymore. Updating h->flag now is racy, - // but it's okay because this is the only possible store - // after creation. - flag = h->flag; - if(flag & IndirectKey) - flag &= ~CanFreeKey; + // eagerly anymore. + flags = h->flags; + if(flags & IndirectKey) + flags &= ~CanFreeKey; else - flag &= ~CanFreeTable; - h->flag = flag; + flags &= ~CanFreeBucket; + h->flags = flags; } it = runtime·mal(sizeof *it); @@ -1165,12 +1303,12 @@ runtime·mapiternext(struct hash_iter *it) if(runtime·gcwaiting) runtime·gosched(); - it->data = hash_next(it); + hash_next(it); if(debug) { runtime·prints("runtime.mapiternext: iter="); runtime·printpointer(it); - runtime·prints("; data="); - runtime·printpointer(it->data); + runtime·prints("; key="); + runtime·printpointer(it->key); runtime·prints("\n"); } } @@ -1188,25 +1326,23 @@ reflect·mapiternext(struct hash_iter *it) void runtime·mapiter1(struct hash_iter *it, ...) { - Hmap *h; byte *ak, *res; Type *key; - h = it->h; ak = (byte*)(&it + 1); - res = it->data; + res = it->key; if(res == nil) runtime·throw("runtime.mapiter1: key:val nil pointer"); key = it->t->key; - key->alg->copy(key->size, ak, hash_keyptr(h, res)); + key->alg->copy(key->size, ak, res); if(debug) { - runtime·prints("mapiter2: iter="); + runtime·prints("mapiter1: iter="); runtime·printpointer(it); runtime·prints("; map="); - runtime·printpointer(h); + runtime·printpointer(it->h); runtime·prints("\n"); } } @@ -1217,11 +1353,11 @@ runtime·mapiterkey(struct hash_iter *it, void *ak) byte *res; Type *key; - res = it->data; + res = it->key; if(res == nil) return false; key = it->t->key; - key->alg->copy(key->size, ak, hash_keyptr(it->h, res)); + key->alg->copy(key->size, ak, res); return true; } @@ -1237,14 +1373,13 @@ reflect·mapiterkey(struct hash_iter *it, uintptr key, bool ok) key = 0; ok = false; - res = it->data; + res = it->key; if(res == nil) { key = 0; ok = false; } else { tkey = it->t->key; key = 0; - res = (byte*)hash_keyptr(it->h, res); if(tkey->size <= sizeof(key)) tkey->alg->copy(tkey->size, (byte*)&key, res); else @@ -1276,7 +1411,6 @@ reflect·maplen(Hmap *h, intgo len) void runtime·mapiter2(struct hash_iter *it, ...) { - Hmap *h; byte *ak, *av, *res; MapType *t; @@ -1284,19 +1418,18 @@ runtime·mapiter2(struct hash_iter *it, ...) ak = (byte*)(&it + 1); av = ak + ROUND(t->key->size, t->elem->align); - res = it->data; + res = it->key; if(res == nil) runtime·throw("runtime.mapiter2: key:val nil pointer"); - h = it->h; - t->key->alg->copy(t->key->size, ak, hash_keyptr(h, res)); - t->elem->alg->copy(t->elem->size, av, hash_valptr(h, res)); + t->key->alg->copy(t->key->size, ak, res); + t->elem->alg->copy(t->elem->size, av, it->value); if(debug) { runtime·prints("mapiter2: iter="); runtime·printpointer(it); runtime·prints("; map="); - runtime·printpointer(h); + runtime·printpointer(it->h); runtime·prints("\n"); } } diff --git a/src/pkg/runtime/hashmap.h b/src/pkg/runtime/hashmap.h index 9b82f299e..2988417f6 100644 --- a/src/pkg/runtime/hashmap.h +++ b/src/pkg/runtime/hashmap.h @@ -2,180 +2,28 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -/* A hash table. - Example, hashing nul-terminated char*s: - hash_hash_t str_hash (void *v) { - char *s; - hash_hash_t hash = 0; - for (s = *(char **)v; *s != 0; s++) { - hash = (hash ^ *s) * 2654435769U; - } - return (hash); - } - int str_eq (void *a, void *b) { - return (strcmp (*(char **)a, *(char **)b) == 0); - } - void str_del (void *arg, void *data) { - *(char **)arg = *(char **)data; - } - - struct hash *h = hash_new (sizeof (char *), &str_hash, &str_eq, &str_del, 3, 12, 15); - ... 3=> 2**3 entries initial size - ... 12=> 2**12 entries before sprouting sub-tables - ... 15=> number of adjacent probes to attempt before growing - - Example lookup: - char *key = "foobar"; - char **result_ptr; - if (hash_lookup (h, &key, (void **) &result_ptr)) { - printf ("found in table: %s\n", *result_ptr); - } else { - printf ("not found in table\n"); - } - - Example insertion: - char *key = strdup ("foobar"); - char **result_ptr; - if (hash_lookup (h, &key, (void **) &result_ptr)) { - printf ("found in table: %s\n", *result_ptr); - printf ("to overwrite, do *result_ptr = key\n"); - } else { - printf ("not found in table; inserted as %s\n", *result_ptr); - assert (*result_ptr == key); - } - - Example deletion: - char *key = "foobar"; - char *result; - if (hash_remove (h, &key, &result)) { - printf ("key found and deleted from table\n"); - printf ("called str_del (&result, data) to copy data to result: %s\n", result); - } else { - printf ("not found in table\n"); - } - - Example iteration over the elements of *h: - char **data; - struct hash_iter it; - hash_iter_init (h, &it); - for (data = hash_next (&it); data != 0; data = hash_next (&it)) { - printf ("%s\n", *data); - } - */ - -#define memset(a,b,c) runtime·memclr((byte*)(a), (uint32)(c)) -#define memcpy(a,b,c) runtime·memmove((byte*)(a),(byte*)(b),(uint32)(c)) -#define assert(a) if(!(a)) runtime·throw("hashmap assert") -#define free(x) runtime·free(x) -#define memmove(a,b,c) runtime·memmove(a, b, c) - struct Hmap; /* opaque */ -struct hash_subtable; /* opaque */ -struct hash_entry; /* opaque */ - -typedef uintptr uintptr_t; -typedef uintptr_t hash_hash_t; - -struct hash_iter { - uint8* data; /* returned from next */ - int32 elemsize; /* size of elements in table */ - int32 changes; /* number of changes observed last time */ - int32 i; /* stack pointer in subtable_state */ - bool cycled; /* have reached the end and wrapped to 0 */ - hash_hash_t last_hash; /* last hash value returned */ - hash_hash_t cycle; /* hash value where we started */ - struct Hmap *h; /* the hash table */ - MapType *t; /* the map type */ - struct hash_iter_sub { - struct hash_entry *e; /* pointer into subtable */ - struct hash_entry *start; /* start of subtable */ - struct hash_entry *last; /* last entry in subtable */ - } subtable_state[4]; /* Should be large enough unless the hashing is - so bad that many distinct data values hash - to the same hash value. */ -}; - -/* Return a hashtable h 2**init_power empty entries, each with - "datasize" data bytes. - (*data_hash)(a) should return the hash value of data element *a. - (*data_eq)(a,b) should return whether the data at "a" and the data at "b" - are equal. - (*data_del)(arg, a) will be invoked when data element *a is about to be removed - from the table. "arg" is the argument passed to "hash_remove()". - - Growing is accomplished by resizing if the current tables size is less than - a threshold, and by adding subtables otherwise. hint should be set - the expected maximum size of the table. - "datasize" should be in [sizeof (void*), ..., 255]. If you need a - bigger "datasize", store a pointer to another piece of memory. */ - -//struct hash *hash_new (int32 datasize, -// hash_hash_t (*data_hash) (void *), -// int32 (*data_eq) (void *, void *), -// void (*data_del) (void *, void *), -// int64 hint); - -/* Lookup *data in *h. If the data is found, return 1 and place a pointer to - the found element in *pres. Otherwise return 0 and place 0 in *pres. */ -// int32 hash_lookup (struct hash *h, void *data, void **pres); - -/* Lookup *data in *h. If the data is found, execute (*data_del) (arg, p) - where p points to the data in the table, then remove it from *h and return - 1. Otherwise return 0. */ -// int32 hash_remove (struct hash *h, void *data, void *arg); - -/* Lookup *data in *h. If the data is found, return 1, and place a pointer - to the found element in *pres. Otherwise, return 0, allocate a region - for the data to be inserted, and place a pointer to the inserted element - in *pres; it is the caller's responsibility to copy the data to be - inserted to the pointer returned in *pres in this case. - - If using garbage collection, it is the caller's responsibility to - add references for **pres if HASH_ADDED is returned. */ -// int32 hash_insert (struct hash *h, void *data, void **pres); - -/* Return the number of elements in the table. */ -// uint32 hash_count (struct hash *h); - -/* The following call is useful only if not using garbage collection on the - table. - Remove all sub-tables associated with *h. - This undoes the effects of hash_init(). - If other memory pointed to by user data must be freed, the caller is - responsible for doing so by iterating over *h first; see - hash_iter_init()/hash_next(). */ -// void hash_destroy (struct hash *h); - -/*----- iteration -----*/ - -/* Initialize *it from *h. */ -// void hash_iter_init (struct hash *h, struct hash_iter *it); - -/* Return the next used entry in the table with which *it was initialized. */ -// void *hash_next (struct hash_iter *it); - -/*---- test interface ----*/ -/* Call (*data_visit) (arg, level, data) for every data entry in the table, - whether used or not. "level" is the subtable level, 0 means first level. */ -/* TESTING ONLY: DO NOT USE THIS ROUTINE IN NORMAL CODE */ -// void hash_visit (struct hash *h, void (*data_visit) (void *arg, int32 level, void *data), void *arg); /* Used by the garbage collector */ struct hash_gciter { - int32 elemsize; - uint8 flag; - uint8 valoff; - uint32 i; /* stack pointer in subtable_state */ - struct hash_subtable *st; - struct hash_gciter_sub { - struct hash_entry *e; /* pointer into subtable */ - struct hash_entry *last; /* last entry in subtable */ - } subtable_state[4]; + Hmap *h; + int32 phase; + uintptr bucket; + struct Bucket *b; + uintptr i; }; + +// this data is used by the garbage collector to keep the map's +// internal structures from being reclaimed. The iterator must +// return in st every live object (ones returned by mallocgc) so +// that those objects won't be collected, and it must return +// every key & value in key_data/val_data so they can get scanned +// for pointers they point to. Note that if you malloc storage +// for keys and values, you need to do both. struct hash_gciter_data { - struct hash_subtable *st; /* subtable pointer, or nil */ + uint8 *st; /* internal structure, or nil */ uint8 *key_data; /* key data, or nil */ uint8 *val_data; /* value data, or nil */ bool indirectkey; /* storing pointers to keys */ diff --git a/src/pkg/runtime/hashmap_fast.c b/src/pkg/runtime/hashmap_fast.c new file mode 100644 index 000000000..2169f4c30 --- /dev/null +++ b/src/pkg/runtime/hashmap_fast.c @@ -0,0 +1,149 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Fast hashmap lookup specialized to a specific key type. +// Included by hashmap.c once for each specialized type. + +// Note that this code differs from hash_lookup in that +// it returns a pointer to the result, not the result itself. +// The returned pointer is only valid until the next GC +// point, so the caller must dereference it before then. + +// +build ignore + +#pragma textflag 7 +void +HASH_LOOKUP1(MapType *t, Hmap *h, KEYTYPE key, byte *value) +{ + uintptr hash; + uintptr bucket; + Bucket *b; + uint8 top; + uintptr i; + KEYTYPE *k; + byte *v; + + if(debug) { + runtime·prints("runtime.mapaccess1_fastXXX: map="); + runtime·printpointer(h); + runtime·prints("; key="); + t->key->alg->print(t->key->size, &key); + runtime·prints("\n"); + } + if(h == nil || h->count == 0) { + value = empty_value; + FLUSH(&value); + return; + } + if(raceenabled) + runtime·racereadpc(h, runtime·getcallerpc(&t), HASH_LOOKUP1); + if(docheck) + check(t, h); + + if(h->B == 0 && (h->count == 1 || QUICKEQ(key))) { + // One-bucket table. Don't hash, just check each bucket entry. + b = (Bucket*)h->buckets; + for(i = 0, k = (KEYTYPE*)b->data, v = (byte*)(k + BUCKETSIZE); i < BUCKETSIZE; i++, k++, v += h->valuesize) { + if(b->tophash[i] != 0 && EQFUNC(key, *k)) { + value = v; + FLUSH(&value); + return; + } + } + } else { + hash = h->hash0; + HASHFUNC(&hash, sizeof(KEYTYPE), &key); + bucket = hash & (((uintptr)1 << h->B) - 1); + if(h->oldbuckets != nil) + grow_work(t, h, bucket); + b = (Bucket*)(h->buckets + bucket * (offsetof(Bucket, data[0]) + BUCKETSIZE * sizeof(KEYTYPE) + BUCKETSIZE * h->valuesize)); + top = hash >> (sizeof(uintptr)*8 - 8); + if(top == 0) + top = 1; + do { + for(i = 0, k = (KEYTYPE*)b->data, v = (byte*)(k + BUCKETSIZE); i < BUCKETSIZE; i++, k++, v += h->valuesize) { + if(b->tophash[i] == top && EQFUNC(key, *k)) { + value = v; + FLUSH(&value); + return; + } + } + b = b->overflow; + } while(b != nil); + } + value = empty_value; + FLUSH(&value); +} + +#pragma textflag 7 +void +HASH_LOOKUP2(MapType *t, Hmap *h, KEYTYPE key, byte *value, bool res) +{ + uintptr hash; + uintptr bucket; + Bucket *b; + uint8 top; + uintptr i; + KEYTYPE *k; + byte *v; + + if(debug) { + runtime·prints("runtime.mapaccess2_fastXXX: map="); + runtime·printpointer(h); + runtime·prints("; key="); + t->key->alg->print(t->key->size, &key); + runtime·prints("\n"); + } + if(h == nil || h->count == 0) { + value = empty_value; + res = false; + FLUSH(&value); + FLUSH(&res); + return; + } + if(raceenabled) + runtime·racereadpc(h, runtime·getcallerpc(&t), HASH_LOOKUP2); + if(docheck) + check(t, h); + + if(h->B == 0 && (h->count == 1 || QUICKEQ(key))) { + // One-bucket table. Don't hash, just check each bucket entry. + b = (Bucket*)h->buckets; + for(i = 0, k = (KEYTYPE*)b->data, v = (byte*)(k + BUCKETSIZE); i < BUCKETSIZE; i++, k++, v += h->valuesize) { + if(b->tophash[i] != 0 && EQFUNC(key, *k)) { + value = v; + res = true; + FLUSH(&value); + FLUSH(&res); + return; + } + } + } else { + hash = h->hash0; + HASHFUNC(&hash, sizeof(KEYTYPE), &key); + bucket = hash & (((uintptr)1 << h->B) - 1); + if(h->oldbuckets != nil) + grow_work(t, h, bucket); + b = (Bucket*)(h->buckets + bucket * (offsetof(Bucket, data[0]) + BUCKETSIZE * sizeof(KEYTYPE) + BUCKETSIZE * h->valuesize)); + top = hash >> (sizeof(uintptr)*8 - 8); + if(top == 0) + top = 1; + do { + for(i = 0, k = (KEYTYPE*)b->data, v = (byte*)(k + BUCKETSIZE); i < BUCKETSIZE; i++, k++, v += h->valuesize) { + if(b->tophash[i] == top && EQFUNC(key, *k)) { + value = v; + res = true; + FLUSH(&value); + FLUSH(&res); + return; + } + } + b = b->overflow; + } while(b != nil); + } + value = empty_value; + res = false; + FLUSH(&value); + FLUSH(&res); +} diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc index ac131b3af..fa28e2b73 100644 --- a/src/pkg/runtime/malloc.goc +++ b/src/pkg/runtime/malloc.goc @@ -35,7 +35,7 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed) MSpan *s; void *v; - if(runtime·gcwaiting && g != m->g0 && m->locks == 0) + if(runtime·gcwaiting && g != m->g0 && m->locks == 0 && dogc) runtime·gosched(); if(m->mallocing) runtime·throw("malloc/free - deadlock"); @@ -516,7 +516,7 @@ runtime·settype_flush(M *mp, bool sysalloc) nbytes3 = 8*sizeof(uintptr) + 1*ntypes; if(!sysalloc) { - data3 = runtime·mallocgc(nbytes3, FlagNoPointers, 0, 1); + data3 = runtime·mallocgc(nbytes3, FlagNoProfiling|FlagNoPointers, 0, 1); } else { data3 = runtime·SysAlloc(nbytes3); if(data3 == nil) @@ -554,7 +554,7 @@ runtime·settype_flush(M *mp, bool sysalloc) nbytes2 = ntypes * sizeof(uintptr); if(!sysalloc) { - data2 = runtime·mallocgc(nbytes2, FlagNoPointers, 0, 1); + data2 = runtime·mallocgc(nbytes2, FlagNoProfiling|FlagNoPointers, 0, 1); } else { data2 = runtime·SysAlloc(nbytes2); if(data2 == nil) diff --git a/src/pkg/runtime/map_test.go b/src/pkg/runtime/map_test.go new file mode 100644 index 000000000..29e19db2c --- /dev/null +++ b/src/pkg/runtime/map_test.go @@ -0,0 +1,282 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "fmt" + "math" + "runtime" + "sort" + "testing" +) + +// negative zero is a good test because: +// 1) 0 and -0 are equal, yet have distinct representations. +// 2) 0 is represented as all zeros, -0 isn't. +// I'm not sure the language spec actually requires this behavior, +// but it's what the current map implementation does. +func TestNegativeZero(t *testing.T) { + m := make(map[float64]bool, 0) + + m[+0.0] = true + m[math.Copysign(0.0, -1.0)] = true // should overwrite +0 entry + + if len(m) != 1 { + t.Error("length wrong") + } + + for k, _ := range m { + if math.Copysign(1.0, k) > 0 { + t.Error("wrong sign") + } + } + + m = make(map[float64]bool, 0) + m[math.Copysign(0.0, -1.0)] = true + m[+0.0] = true // should overwrite -0.0 entry + + if len(m) != 1 { + t.Error("length wrong") + } + + for k, _ := range m { + if math.Copysign(1.0, k) < 0 { + t.Error("wrong sign") + } + } +} + +// nan is a good test because nan != nan, and nan has +// a randomized hash value. +func TestNan(t *testing.T) { + m := make(map[float64]int, 0) + nan := math.NaN() + m[nan] = 1 + m[nan] = 2 + m[nan] = 4 + if len(m) != 3 { + t.Error("length wrong") + } + s := 0 + for k, v := range m { + if k == k { + t.Error("nan disappeared") + } + if (v & (v - 1)) != 0 { + t.Error("value wrong") + } + s |= v + } + if s != 7 { + t.Error("values wrong") + } +} + +// Maps aren't actually copied on assignment. +func TestAlias(t *testing.T) { + m := make(map[int]int, 0) + m[0] = 5 + n := m + n[0] = 6 + if m[0] != 6 { + t.Error("alias didn't work") + } +} + +func TestGrowWithNaN(t *testing.T) { + m := make(map[float64]int, 4) + nan := math.NaN() + m[nan] = 1 + m[nan] = 2 + m[nan] = 4 + cnt := 0 + s := 0 + growflag := true + for k, v := range m { + if growflag { + // force a hashtable resize + for i := 0; i < 100; i++ { + m[float64(i)] = i + } + growflag = false + } + if k != k { + cnt++ + s |= v + } + } + if cnt != 3 { + t.Error("NaN keys lost during grow") + } + if s != 7 { + t.Error("NaN values lost during grow") + } +} + +type FloatInt struct { + x float64 + y int +} + +func TestGrowWithNegativeZero(t *testing.T) { + negzero := math.Copysign(0.0, -1.0) + m := make(map[FloatInt]int, 4) + m[FloatInt{0.0, 0}] = 1 + m[FloatInt{0.0, 1}] = 2 + m[FloatInt{0.0, 2}] = 4 + m[FloatInt{0.0, 3}] = 8 + growflag := true + s := 0 + cnt := 0 + negcnt := 0 + // The first iteration should return the +0 key. + // The subsequent iterations should return the -0 key. + // I'm not really sure this is required by the spec, + // but it makes sense. + // TODO: are we allowed to get the first entry returned again??? + for k, v := range m { + if v == 0 { + continue + } // ignore entries added to grow table + cnt++ + if math.Copysign(1.0, k.x) < 0 { + if v&16 == 0 { + t.Error("key/value not updated together 1") + } + negcnt++ + s |= v & 15 + } else { + if v&16 == 16 { + t.Error("key/value not updated together 2", k, v) + } + s |= v + } + if growflag { + // force a hashtable resize + for i := 0; i < 100; i++ { + m[FloatInt{3.0, i}] = 0 + } + // then change all the entries + // to negative zero + m[FloatInt{negzero, 0}] = 1 | 16 + m[FloatInt{negzero, 1}] = 2 | 16 + m[FloatInt{negzero, 2}] = 4 | 16 + m[FloatInt{negzero, 3}] = 8 | 16 + growflag = false + } + } + if s != 15 { + t.Error("entry missing", s) + } + if cnt != 4 { + t.Error("wrong number of entries returned by iterator", cnt) + } + if negcnt != 3 { + t.Error("update to negzero missed by iteration", negcnt) + } +} + +func TestIterGrowAndDelete(t *testing.T) { + m := make(map[int]int, 4) + for i := 0; i < 100; i++ { + m[i] = i + } + growflag := true + for k := range m { + if growflag { + // grow the table + for i := 100; i < 1000; i++ { + m[i] = i + } + // delete all odd keys + for i := 1; i < 1000; i += 2 { + delete(m, i) + } + growflag = false + } else { + if k&1 == 1 { + t.Error("odd value returned") + } + } + } +} + +// make sure old bucket arrays don't get GCd while +// an iterator is still using them. +func TestIterGrowWithGC(t *testing.T) { + m := make(map[int]int, 4) + for i := 0; i < 16; i++ { + m[i] = i + } + growflag := true + bitmask := 0 + for k := range m { + if k < 16 { + bitmask |= 1 << uint(k) + } + if growflag { + // grow the table + for i := 100; i < 1000; i++ { + m[i] = i + } + // trigger a gc + runtime.GC() + growflag = false + } + } + if bitmask != 1<<16-1 { + t.Error("missing key", bitmask) + } +} + +func TestBigItems(t *testing.T) { + var key [256]string + for i := 0; i < 256; i++ { + key[i] = "foo" + } + m := make(map[[256]string][256]string, 4) + for i := 0; i < 100; i++ { + key[37] = fmt.Sprintf("string%02d", i) + m[key] = key + } + var keys [100]string + var values [100]string + i := 0 + for k, v := range m { + keys[i] = k[37] + values[i] = v[37] + i++ + } + sort.Strings(keys[:]) + sort.Strings(values[:]) + for i := 0; i < 100; i++ { + if keys[i] != fmt.Sprintf("string%02d", i) { + t.Errorf("#%d: missing key: %v", i, keys[i]) + } + if values[i] != fmt.Sprintf("string%02d", i) { + t.Errorf("#%d: missing value: %v", i, values[i]) + } + } +} + +type empty struct { +} + +func TestEmptyKeyAndValue(t *testing.T) { + a := make(map[int]empty, 4) + b := make(map[empty]int, 4) + c := make(map[empty]empty, 4) + a[0] = empty{} + b[empty{}] = 0 + b[empty{}] = 1 + c[empty{}] = empty{} + + if len(a) != 1 { + t.Errorf("empty value insert problem") + } + if b[empty{}] != 1 { + t.Errorf("empty key returned wrong value") + } +} diff --git a/src/pkg/runtime/mapspeed_test.go b/src/pkg/runtime/mapspeed_test.go new file mode 100644 index 000000000..a37974060 --- /dev/null +++ b/src/pkg/runtime/mapspeed_test.go @@ -0,0 +1,150 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package runtime_test + +import ( + "fmt" + "strings" + "testing" +) + +const size = 10 + +func BenchmarkHashStringSpeed(b *testing.B) { + strings := make([]string, size) + for i := 0; i < size; i++ { + strings[i] = fmt.Sprintf("string#%d", i) + } + sum := 0 + m := make(map[string]int, size) + for i := 0; i < size; i++ { + m[strings[i]] = 0 + } + idx := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + sum += m[strings[idx]] + idx++ + if idx == size { + idx = 0 + } + } +} + +func BenchmarkHashInt32Speed(b *testing.B) { + ints := make([]int32, size) + for i := 0; i < size; i++ { + ints[i] = int32(i) + } + sum := 0 + m := make(map[int32]int, size) + for i := 0; i < size; i++ { + m[ints[i]] = 0 + } + idx := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + sum += m[ints[idx]] + idx++ + if idx == size { + idx = 0 + } + } +} + +func BenchmarkHashInt64Speed(b *testing.B) { + ints := make([]int64, size) + for i := 0; i < size; i++ { + ints[i] = int64(i) + } + sum := 0 + m := make(map[int64]int, size) + for i := 0; i < size; i++ { + m[ints[i]] = 0 + } + idx := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + sum += m[ints[idx]] + idx++ + if idx == size { + idx = 0 + } + } +} +func BenchmarkHashStringArraySpeed(b *testing.B) { + stringpairs := make([][2]string, size) + for i := 0; i < size; i++ { + for j := 0; j < 2; j++ { + stringpairs[i][j] = fmt.Sprintf("string#%d/%d", i, j) + } + } + sum := 0 + m := make(map[[2]string]int, size) + for i := 0; i < size; i++ { + m[stringpairs[i]] = 0 + } + idx := 0 + b.ResetTimer() + for i := 0; i < b.N; i++ { + sum += m[stringpairs[idx]] + idx++ + if idx == size { + idx = 0 + } + } +} + +func BenchmarkMegMap(b *testing.B) { + m := make(map[string]bool) + for suffix := 'A'; suffix <= 'G'; suffix++ { + m[strings.Repeat("X", 1<<20-1)+fmt.Sprint(suffix)] = true + } + key := strings.Repeat("X", 1<<20-1) + "k" + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = m[key] + } +} + +func BenchmarkMegOneMap(b *testing.B) { + m := make(map[string]bool) + m[strings.Repeat("X", 1<<20)] = true + key := strings.Repeat("Y", 1<<20) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = m[key] + } +} + +func BenchmarkMegEmptyMap(b *testing.B) { + m := make(map[string]bool) + key := strings.Repeat("X", 1<<20) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = m[key] + } +} + +func BenchmarkSmallStrMap(b *testing.B) { + m := make(map[string]bool) + for suffix := 'A'; suffix <= 'G'; suffix++ { + m[fmt.Sprint(suffix)] = true + } + key := "k" + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = m[key] + } +} +func BenchmarkIntMap(b *testing.B) { + m := make(map[int]bool) + for i := 0; i < 8; i++ { + m[i] = true + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = m[7] + } +} diff --git a/src/pkg/runtime/mcentral.c b/src/pkg/runtime/mcentral.c index ac8b5aa0d..ec2a91ad5 100644 --- a/src/pkg/runtime/mcentral.c +++ b/src/pkg/runtime/mcentral.c @@ -19,7 +19,6 @@ #include "malloc.h" static bool MCentral_Grow(MCentral *c); -static void* MCentral_Alloc(MCentral *c); static void MCentral_Free(MCentral *c, void *v); // Initialize a single central free list. diff --git a/src/pkg/runtime/mem_darwin.c b/src/pkg/runtime/mem_darwin.c index 04e719394..7aa607f8e 100644 --- a/src/pkg/runtime/mem_darwin.c +++ b/src/pkg/runtime/mem_darwin.c @@ -37,7 +37,12 @@ runtime·SysFree(void *v, uintptr n) void* runtime·SysReserve(void *v, uintptr n) { - return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); + void *p; + + p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); + if(p < (void*)4096) + return nil; + return p; } enum @@ -52,7 +57,7 @@ runtime·SysMap(void *v, uintptr n) mstats.sys += n; p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) runtime·throw("runtime: cannot map pages in arena address space"); diff --git a/src/pkg/runtime/mem_freebsd.c b/src/pkg/runtime/mem_freebsd.c index f217e9db1..805e74cff 100644 --- a/src/pkg/runtime/mem_freebsd.c +++ b/src/pkg/runtime/mem_freebsd.c @@ -8,6 +8,11 @@ #include "os_GOOS.h" #include "malloc.h" +enum +{ + ENOMEM = 12, +}; + void* runtime·SysAlloc(uintptr n) { @@ -36,20 +41,20 @@ runtime·SysFree(void *v, uintptr n) void* runtime·SysReserve(void *v, uintptr n) { + void *p; + // On 64-bit, people with ulimit -v set complain if we reserve too // much address space. Instead, assume that the reservation is okay // and check the assumption in SysMap. if(sizeof(void*) == 8) return v; - return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); + p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); + if(p < (void*)4096) + return nil; + return p; } -enum -{ - ENOMEM = 12, -}; - void runtime·SysMap(void *v, uintptr n) { @@ -60,7 +65,7 @@ runtime·SysMap(void *v, uintptr n) // On 64-bit, we don't actually have v reserved, so tread carefully. if(sizeof(void*) == 8) { p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) { runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); @@ -70,7 +75,7 @@ runtime·SysMap(void *v, uintptr n) } p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) runtime·throw("runtime: cannot map pages in arena address space"); diff --git a/src/pkg/runtime/mem_linux.c b/src/pkg/runtime/mem_linux.c index ebcec1e86..1bae755fa 100644 --- a/src/pkg/runtime/mem_linux.c +++ b/src/pkg/runtime/mem_linux.c @@ -10,8 +10,6 @@ enum { - EAGAIN = 11, - ENOMEM = 12, _PAGE_SIZE = 4096, }; diff --git a/src/pkg/runtime/mem_netbsd.c b/src/pkg/runtime/mem_netbsd.c index 77ce04c4e..e5bdac0ef 100644 --- a/src/pkg/runtime/mem_netbsd.c +++ b/src/pkg/runtime/mem_netbsd.c @@ -50,10 +50,9 @@ runtime·SysReserve(void *v, uintptr n) return v; p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if (p == ((void *)-ENOMEM)) + if(p < (void*)4096) return nil; - else - return p; + return p; } void @@ -66,7 +65,7 @@ runtime·SysMap(void *v, uintptr n) // On 64-bit, we don't actually have v reserved, so tread carefully. if(sizeof(void*) == 8) { p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) { runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); @@ -76,7 +75,7 @@ runtime·SysMap(void *v, uintptr n) } p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) runtime·throw("runtime: cannot map pages in arena address space"); diff --git a/src/pkg/runtime/mem_openbsd.c b/src/pkg/runtime/mem_openbsd.c index 77ce04c4e..e5bdac0ef 100644 --- a/src/pkg/runtime/mem_openbsd.c +++ b/src/pkg/runtime/mem_openbsd.c @@ -50,10 +50,9 @@ runtime·SysReserve(void *v, uintptr n) return v; p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0); - if (p == ((void *)-ENOMEM)) + if(p < (void*)4096) return nil; - else - return p; + return p; } void @@ -66,7 +65,7 @@ runtime·SysMap(void *v, uintptr n) // On 64-bit, we don't actually have v reserved, so tread carefully. if(sizeof(void*) == 8) { p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) { runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p); @@ -76,7 +75,7 @@ runtime·SysMap(void *v, uintptr n) } p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0); - if(p == (void*)-ENOMEM) + if(p == (void*)ENOMEM) runtime·throw("runtime: out of memory"); if(p != v) runtime·throw("runtime: cannot map pages in arena address space"); diff --git a/src/pkg/runtime/memmove_amd64.s b/src/pkg/runtime/memmove_amd64.s index e78be8145..6174407e3 100644 --- a/src/pkg/runtime/memmove_amd64.s +++ b/src/pkg/runtime/memmove_amd64.s @@ -23,11 +23,12 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. +// void runtime·memmove(void*, void*, uintptr) TEXT runtime·memmove(SB), 7, $0 MOVQ to+0(FP), DI MOVQ fr+8(FP), SI - MOVLQSX n+16(FP), BX + MOVQ n+16(FP), BX /* * check and set for backwards @@ -38,7 +39,7 @@ TEXT runtime·memmove(SB), 7, $0 /* * forward copy loop */ -forward: +forward: MOVQ BX, CX SHRQ $3, CX ANDQ $7, BX diff --git a/src/pkg/runtime/memmove_linux_amd64_test.go b/src/pkg/runtime/memmove_linux_amd64_test.go new file mode 100644 index 000000000..f7221f4f5 --- /dev/null +++ b/src/pkg/runtime/memmove_linux_amd64_test.go @@ -0,0 +1,61 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "io/ioutil" + "os" + "reflect" + "syscall" + "testing" + "unsafe" +) + +// TestMemmoveOverflow maps 3GB of memory and calls memmove on +// the corresponding slice. +func TestMemmoveOverflow(t *testing.T) { + // Create a temporary file. + tmp, err := ioutil.TempFile("", "go-memmovetest") + if err != nil { + t.Fatal(err) + } + _, err = tmp.Write(make([]byte, 65536)) + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + + // Set up mappings. + base, _, errno := syscall.Syscall6(syscall.SYS_MMAP, + 0xa0<<32, 3<<30, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_PRIVATE|syscall.MAP_ANONYMOUS, ^uintptr(0), 0) + if errno != 0 { + t.Skipf("could not create memory mapping: %s", errno) + } + syscall.Syscall(syscall.SYS_MUNMAP, base, 3<<30, 0) + + for off := uintptr(0); off < 3<<30; off += 65536 { + _, _, errno := syscall.Syscall6(syscall.SYS_MMAP, + base+off, 65536, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED|syscall.MAP_FIXED, tmp.Fd(), 0) + if errno != 0 { + t.Fatalf("could not map a page at requested 0x%x: %s", base+off, errno) + } + defer syscall.Syscall(syscall.SYS_MUNMAP, base+off, 65536, 0) + } + + var s []byte + sp := (*reflect.SliceHeader)(unsafe.Pointer(&s)) + sp.Data = base + sp.Len, sp.Cap = 3<<30, 3<<30 + + n := copy(s[1:], s) + if n != 3<<30-1 { + t.Fatalf("copied %d bytes, expected %d", n, 3<<30-1) + } + n = copy(s, s[1:]) + if n != 3<<30-1 { + t.Fatalf("copied %d bytes, expected %d", n, 3<<30-1) + } +} diff --git a/src/pkg/runtime/mfixalloc.c b/src/pkg/runtime/mfixalloc.c index c916d588f..c7dab8aea 100644 --- a/src/pkg/runtime/mfixalloc.c +++ b/src/pkg/runtime/mfixalloc.c @@ -30,6 +30,11 @@ void* runtime·FixAlloc_Alloc(FixAlloc *f) { void *v; + + if(f->size == 0) { + runtime·printf("runtime: use of FixAlloc_Alloc before FixAlloc_Init\n"); + runtime·throw("runtime: internal error"); + } if(f->list) { v = f->list; diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index 010f9cd96..aa499f476 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -140,6 +140,7 @@ static Workbuf* getempty(Workbuf*); static Workbuf* getfull(Workbuf*); static void putempty(Workbuf*); static Workbuf* handoff(Workbuf*); +static void gchelperstart(void); static struct { uint64 full; // lock-free list of full blocks @@ -191,7 +192,7 @@ static struct { // markonly marks an object. It returns true if the object // has been marked by this function, false otherwise. -// This function isn't thread-safe and doesn't append the object to any buffer. +// This function doesn't append the object to any buffer. static bool markonly(void *obj) { @@ -254,13 +255,23 @@ found: // Only care about allocated and not marked. if((bits & (bitAllocated|bitMarked)) != bitAllocated) return false; - *bitp |= bitMarked<<shift; + if(work.nproc == 1) + *bitp |= bitMarked<<shift; + else { + for(;;) { + x = *bitp; + if(x & (bitMarked<<shift)) + return false; + if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift)))) + break; + } + } // The object is now marked return true; } -// PtrTarget and BitTarget are structures used by intermediate buffers. +// PtrTarget is a structure used by intermediate buffers. // The intermediate buffers hold GC data before it // is moved/flushed to the work buffer (Workbuf). // The size of an intermediate buffer is very small, @@ -272,25 +283,17 @@ struct PtrTarget uintptr ti; }; -typedef struct BitTarget BitTarget; -struct BitTarget -{ - void *p; - uintptr ti; - uintptr *bitp, shift; -}; - typedef struct BufferList BufferList; struct BufferList { PtrTarget ptrtarget[IntermediateBufferCapacity]; - BitTarget bittarget[IntermediateBufferCapacity]; Obj obj[IntermediateBufferCapacity]; - BufferList *next; + uint32 busy; + byte pad[CacheLineSize]; }; -static BufferList *bufferList; +#pragma dataflag 16 // no pointers +static BufferList bufferList[MaxGcproc]; -static Lock lock; static Type *itabtype; static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); @@ -301,7 +304,6 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); // and are prepared to be scanned by the garbage collector. // // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer. -// bitbuf holds temporary data generated by this function. // // A simplified drawing explaining how the todo-list moves from a structure to another: // @@ -309,14 +311,12 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj); // (find pointers) // Obj ------> PtrTarget (pointer targets) // ↑ | -// | | flushptrbuf (1st part, -// | | find block start) -// | ↓ -// `--------- BitTarget (pointer targets and the corresponding locations in bitmap) -// flushptrbuf -// (2nd part, mark and enqueue) +// | | +// `----------' +// flushptrbuf +// (find block start, mark and enqueue) static void -flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, BitTarget *bitbuf) +flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj) { byte *p, *arena_start, *obj; uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n; @@ -325,7 +325,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf Obj *wp; Workbuf *wbuf; PtrTarget *ptrbuf_end; - BitTarget *bitbufpos, *bt; arena_start = runtime·mheap->arena_start; @@ -359,8 +358,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf { // Multi-threaded version. - bitbufpos = bitbuf; - while(ptrbuf < ptrbuf_end) { obj = ptrbuf->p; ti = ptrbuf->ti; @@ -438,26 +435,22 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf // Only care about allocated and not marked. if((bits & (bitAllocated|bitMarked)) != bitAllocated) continue; - - *bitbufpos++ = (BitTarget){obj, ti, bitp, shift}; - } - - runtime·lock(&lock); - for(bt=bitbuf; bt<bitbufpos; bt++){ - xbits = *bt->bitp; - bits = xbits >> bt->shift; - if((bits & bitMarked) != 0) - continue; - - // Mark the block - *bt->bitp = xbits | (bitMarked << bt->shift); + if(work.nproc == 1) + *bitp |= bitMarked<<shift; + else { + for(;;) { + x = *bitp; + if(x & (bitMarked<<shift)) + goto continue_obj; + if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift)))) + break; + } + } // If object has no pointers, don't need to scan further. if((bits & bitNoPointers) != 0) continue; - obj = bt->p; - // Ask span about size class. // (Manually inlined copy of MHeap_Lookup.) x = (uintptr)obj >> PageShift; @@ -467,11 +460,11 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf PREFETCH(obj); - *wp = (Obj){obj, s->elemsize, bt->ti}; + *wp = (Obj){obj, s->elemsize, ti}; wp++; nobj++; + continue_obj:; } - runtime·unlock(&lock); // If another proc wants a pointer, give it some. if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) { @@ -575,20 +568,19 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) byte *b, *arena_start, *arena_used; uintptr n, i, end_b, elemsize, size, ti, objti, count, type; uintptr *pc, precise_type, nominal_size; - uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti; + uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ret; void *obj; Type *t; Slice *sliceptr; Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4]; BufferList *scanbuffers; PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos; - BitTarget *bitbuf; Obj *objbuf, *objbuf_end, *objbufpos; Eface *eface; Iface *iface; Hmap *hmap; MapType *maptype; - bool didmark, mapkey_kind, mapval_kind; + bool mapkey_kind, mapval_kind; struct hash_gciter map_iter; struct hash_gciter_data d; Hchan *chan; @@ -606,26 +598,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) precise_type = false; nominal_size = 0; - // Allocate ptrbuf, bitbuf + // Allocate ptrbuf { - runtime·lock(&lock); - - if(bufferList == nil) { - bufferList = runtime·SysAlloc(sizeof(*bufferList)); - if(bufferList == nil) - runtime·throw("runtime: cannot allocate memory"); - bufferList->next = nil; - } - scanbuffers = bufferList; - bufferList = bufferList->next; - + scanbuffers = &bufferList[m->helpgc]; ptrbuf = &scanbuffers->ptrtarget[0]; ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget); - bitbuf = &scanbuffers->bittarget[0]; objbuf = &scanbuffers->obj[0]; objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj); - - runtime·unlock(&lock); } ptrbufpos = ptrbuf; @@ -638,6 +617,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) mapkey_ti = mapval_ti = 0; chan = nil; chantype = nil; + chan_ret = nil; goto next_block; @@ -703,7 +683,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) mapval_kind = maptype->elem->kind; mapval_ti = (uintptr)maptype->elem->gc | PRECISE; - map_ret = 0; + map_ret = nil; pc = mapProg; } else { goto next_block; @@ -712,6 +692,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) case TypeInfo_Chan: chan = (Hchan*)b; chantype = (ChanType*)t; + chan_ret = nil; pc = chanProg; break; default: @@ -759,17 +740,29 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) case GC_STRING: obj = *(void**)(stack_top.b + pc[1]); + markonly(obj); pc += 2; - break; + continue; case GC_EFACE: eface = (Eface*)(stack_top.b + pc[1]); pc += 2; - if(eface->type != nil && (eface->data >= arena_start && eface->data < arena_used)) { - t = eface->type; + if(eface->type == nil) + continue; + + // eface->type + t = eface->type; + if((void*)t >= arena_start && (void*)t < arena_used) { + *ptrbufpos++ = (PtrTarget){t, 0}; + if(ptrbufpos == ptrbuf_end) + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); + } + + // eface->data + if(eface->data >= arena_start && eface->data < arena_used) { if(t->size <= sizeof(void*)) { if((t->kind & KindNoPointers)) - break; + continue; obj = eface->data; if((t->kind & ~KindNoPointers) == KindPtr) @@ -785,13 +778,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) iface = (Iface*)(stack_top.b + pc[1]); pc += 2; if(iface->tab == nil) - break; + continue; // iface->tab if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) { *ptrbufpos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc}; if(ptrbufpos == ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); } // iface->data @@ -799,7 +792,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) t = iface->tab->type; if(t->size <= sizeof(void*)) { if((t->kind & KindNoPointers)) - break; + continue; obj = iface->data; if((t->kind & ~KindNoPointers) == KindPtr) @@ -812,13 +805,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) break; case GC_DEFAULT_PTR: - while((i = stack_top.b) <= end_b) { + while(stack_top.b <= end_b) { + obj = *(byte**)stack_top.b; stack_top.b += PtrSize; - obj = *(byte**)i; if(obj >= arena_start && obj < arena_used) { *ptrbufpos++ = (PtrTarget){obj, 0}; if(ptrbufpos == ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); } } goto next_block; @@ -826,9 +819,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) case GC_END: if(--stack_top.count != 0) { // Next iteration of a loop if possible. - elemsize = stack_top.elemsize; - stack_top.b += elemsize; - if(stack_top.b + elemsize <= end_b+PtrSize) { + stack_top.b += stack_top.elemsize; + if(stack_top.b + stack_top.elemsize <= end_b+PtrSize) { pc = stack_top.loop_or_ret; continue; } @@ -894,10 +886,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) pc += 3; continue; } - runtime·lock(&lock); - didmark = markonly(hmap); - runtime·unlock(&lock); - if(didmark) { + if(markonly(hmap)) { maptype = (MapType*)pc[2]; if(hash_gciter_init(hmap, &map_iter)) { mapkey_size = maptype->key->size; @@ -923,31 +912,41 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) while(hash_gciter_next(&map_iter, &d)) { // buffers: reserve space for 2 objects. if(ptrbufpos+2 >= ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); if(objbufpos+2 >= objbuf_end) flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); - if(d.st != nil) { - runtime·lock(&lock); + if(d.st != nil) markonly(d.st); - runtime·unlock(&lock); - } + if(d.key_data != nil) { if(!(mapkey_kind & KindNoPointers) || d.indirectkey) { if(!d.indirectkey) *objbufpos++ = (Obj){d.key_data, mapkey_size, mapkey_ti}; - else + else { + if(Debug) { + obj = *(void**)d.key_data; + if(!(arena_start <= obj && obj < arena_used)) + runtime·throw("scanblock: inconsistent hashmap"); + } *ptrbufpos++ = (PtrTarget){*(void**)d.key_data, mapkey_ti}; + } } if(!(mapval_kind & KindNoPointers) || d.indirectval) { if(!d.indirectval) *objbufpos++ = (Obj){d.val_data, mapval_size, mapval_ti}; - else + else { + if(Debug) { + obj = *(void**)d.val_data; + if(!(arena_start <= obj && obj < arena_used)) + runtime·throw("scanblock: inconsistent hashmap"); + } *ptrbufpos++ = (PtrTarget){*(void**)d.val_data, mapval_ti}; + } } } } - if(map_ret == 0) + if(map_ret == nil) goto next_block; pc = map_ret; continue; @@ -961,7 +960,26 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) *objbufpos++ = (Obj){obj, size, objti}; if(objbufpos == objbuf_end) flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); - break; + continue; + + case GC_CHAN_PTR: + // Similar to GC_MAP_PTR + chan = *(Hchan**)(stack_top.b + pc[1]); + if(chan == nil) { + pc += 3; + continue; + } + if(markonly(chan)) { + chantype = (ChanType*)pc[2]; + if(!(chantype->elem->kind & KindNoPointers)) { + // Start chanProg. + chan_ret = pc+3; + pc = chanProg+1; + continue; + } + } + pc += 3; + continue; case GC_CHAN: // There are no heap pointers in struct Hchan, @@ -981,7 +999,10 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); } } - goto next_block; + if(chan_ret == nil) + goto next_block; + pc = chan_ret; + continue; default: runtime·throw("scanblock: invalid GC instruction"); @@ -991,7 +1012,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) if(obj >= arena_start && obj < arena_used) { *ptrbufpos++ = (PtrTarget){obj, objti}; if(ptrbufpos == ptrbuf_end) - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); } } @@ -1000,7 +1021,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) // the loop by setting b, n, ti to the parameters for the next block. if(nobj == 0) { - flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf); + flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj); flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj); if(nobj == 0) { @@ -1026,11 +1047,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking) nobj--; } -endscan: - runtime·lock(&lock); - scanbuffers->next = bufferList; - bufferList = scanbuffers; - runtime·unlock(&lock); +endscan:; } // debug_scanblock is the debug copy of scanblock. @@ -1339,7 +1356,7 @@ addstackroots(G *gp) runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk); runtime·throw("scanstack"); } - addroot((Obj){sp, (byte*)stk - sp, 0}); + addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg | PRECISE | LOOP}); sp = (byte*)stk->gobuf.sp; guard = stk->stackguard; stk = (Stktop*)stk->stackbase; @@ -1381,14 +1398,17 @@ addroots(void) for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) { s = allspans[spanidx]; if(s->state == MSpanInUse) { + // The garbage collector ignores type pointers stored in MSpan.types: + // - Compiler-generated types are stored outside of heap. + // - The reflect package has runtime-generated types cached in its data structures. + // The garbage collector relies on finding the references via that cache. switch(s->types.compression) { case MTypes_Empty: case MTypes_Single: break; case MTypes_Words: case MTypes_Bytes: - // TODO(atom): consider using defaultProg instead of 0 - addroot((Obj){(byte*)&s->types.data, sizeof(void*), 0}); + markonly((byte*)s->types.data); break; } } @@ -1655,6 +1675,8 @@ runtime·memorydump(void) void runtime·gchelper(void) { + gchelperstart(); + // parallel mark for over gc roots runtime·parfordo(work.markfor); @@ -1668,6 +1690,7 @@ runtime·gchelper(void) } runtime·parfordo(work.sweepfor); + bufferList[m->helpgc].busy = 0; if(runtime·xadd(&work.ndone, +1) == work.nproc-1) runtime·notewakeup(&work.alldone); } @@ -1757,6 +1780,8 @@ runtime·gc(int32 force) // a problem in the past. if((((uintptr)&work.empty) & 7) != 0) runtime·throw("runtime: gc work buffer is misaligned"); + if((((uintptr)&work.full) & 7) != 0) + runtime·throw("runtime: gc work buffer is misaligned"); // The gc is turned off (via enablegc) until // the bootstrap has completed. @@ -1857,6 +1882,7 @@ gc(struct gc_args *args) t1 = runtime·nanotime(); + gchelperstart(); runtime·parfordo(work.markfor); scanblock(nil, nil, 0, true); @@ -1868,6 +1894,7 @@ gc(struct gc_args *args) t2 = runtime·nanotime(); runtime·parfordo(work.sweepfor); + bufferList[m->helpgc].busy = 0; t3 = runtime·nanotime(); if(work.nproc > 1) @@ -2009,6 +2036,15 @@ runtime∕debug·setGCPercent(intgo in, intgo out) } static void +gchelperstart(void) +{ + if(m->helpgc < 0 || m->helpgc >= MaxGcproc) + runtime·throw("gchelperstart: bad m->helpgc"); + if(runtime·xchg(&bufferList[m->helpgc].busy, 1)) + runtime·throw("gchelperstart: already busy"); +} + +static void runfinq(void) { Finalizer *f; diff --git a/src/pkg/runtime/mgc0.h b/src/pkg/runtime/mgc0.h index 87b604a36..18f3654b4 100644 --- a/src/pkg/runtime/mgc0.h +++ b/src/pkg/runtime/mgc0.h @@ -24,6 +24,7 @@ enum { GC_ARRAY_NEXT, // The next element of an array. Args: none GC_CALL, // Call a subroutine. Args: (off, objgcrel) GC_MAP_PTR, // Go map. Args: (off, MapType*) + GC_CHAN_PTR, // Go channel. Args: (off, ChanType*) GC_STRING, // Go string. Args: (off) GC_EFACE, // interface{}. Args: (off) GC_IFACE, // interface{...}. Args: (off) diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c index f45149d63..177f40659 100644 --- a/src/pkg/runtime/mheap.c +++ b/src/pkg/runtime/mheap.c @@ -409,6 +409,9 @@ runtime·MHeap_Scavenger(void) bool trace; Note note, *notep; + g->issystem = true; + g->isbackground = true; + // If we go two minutes without a garbage collection, force one to run. forcegc = 2*60*1e9; // If a span goes unused for 5 minutes after a garbage collection, diff --git a/src/pkg/runtime/netpoll.goc b/src/pkg/runtime/netpoll.goc new file mode 100644 index 000000000..06b6d6172 --- /dev/null +++ b/src/pkg/runtime/netpoll.goc @@ -0,0 +1,351 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin linux + +package net + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "arch_GOARCH.h" +#include "malloc.h" + +// Integrated network poller (platform-independent part). +// A particular implementation (epoll/kqueue) must define the following functions: +// void runtime·netpollinit(void); // to initialize the poller +// int32 runtime·netpollopen(int32 fd, PollDesc *pd); // to arm edge-triggered notifications + // and associate fd with pd. +// An implementation must call the following function to denote that the pd is ready. +// void runtime·netpollready(G **gpp, PollDesc *pd, int32 mode); + +#define READY ((G*)1) + +struct PollDesc +{ + PollDesc* link; // in pollcache, protected by pollcache.Lock + Lock; // protectes the following fields + int32 fd; + bool closing; + uintptr seq; // protects from stale timers and ready notifications + G* rg; // G waiting for read or READY (binary semaphore) + Timer rt; // read deadline timer (set if rt.fv != nil) + int64 rd; // read deadline + G* wg; // the same for writes + Timer wt; + int64 wd; +}; + +static struct +{ + Lock; + PollDesc* first; + // PollDesc objects must be type-stable, + // because we can get ready notification from epoll/kqueue + // after the descriptor is closed/reused. + // Stale notifications are detected using seq variable, + // seq is incremented when deadlines are changed or descriptor is reused. +} pollcache; + +static void netpollblock(PollDesc*, int32); +static G* netpollunblock(PollDesc*, int32); +static void deadline(int64, Eface); +static void readDeadline(int64, Eface); +static void writeDeadline(int64, Eface); +static PollDesc* allocPollDesc(void); +static intgo checkerr(PollDesc *pd, int32 mode); + +static FuncVal deadlineFn = {(void(*)(void))deadline}; +static FuncVal readDeadlineFn = {(void(*)(void))readDeadline}; +static FuncVal writeDeadlineFn = {(void(*)(void))writeDeadline}; + +func runtime_pollServerInit() { + runtime·netpollinit(); +} + +func runtime_pollOpen(fd int) (pd *PollDesc, errno int) { + pd = allocPollDesc(); + runtime·lock(pd); + if(pd->wg != nil && pd->wg != READY) + runtime·throw("runtime_pollOpen: blocked write on free descriptor"); + if(pd->rg != nil && pd->rg != READY) + runtime·throw("runtime_pollOpen: blocked read on free descriptor"); + pd->fd = fd; + pd->closing = false; + pd->seq++; + pd->rg = nil; + pd->rd = 0; + pd->wg = nil; + pd->wd = 0; + runtime·unlock(pd); + + errno = runtime·netpollopen(fd, pd); +} + +func runtime_pollClose(pd *PollDesc) { + if(!pd->closing) + runtime·throw("runtime_pollClose: close w/o unblock"); + if(pd->wg != nil && pd->wg != READY) + runtime·throw("runtime_pollClose: blocked write on closing descriptor"); + if(pd->rg != nil && pd->rg != READY) + runtime·throw("runtime_pollClose: blocked read on closing descriptor"); + runtime·netpollclose(pd->fd); + runtime·lock(&pollcache); + pd->link = pollcache.first; + pollcache.first = pd; + runtime·unlock(&pollcache); +} + +func runtime_pollReset(pd *PollDesc, mode int) (err int) { + runtime·lock(pd); + err = checkerr(pd, mode); + if(err) + goto ret; + if(mode == 'r') + pd->rg = nil; + else if(mode == 'w') + pd->wg = nil; +ret: + runtime·unlock(pd); +} + +func runtime_pollWait(pd *PollDesc, mode int) (err int) { + runtime·lock(pd); + err = checkerr(pd, mode); + if(err) + goto ret; + netpollblock(pd, mode); + err = checkerr(pd, mode); +ret: + runtime·unlock(pd); +} + +func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) { + runtime·lock(pd); + if(pd->closing) + goto ret; + pd->seq++; // invalidate current timers + // Reset current timers. + if(pd->rt.fv) { + runtime·deltimer(&pd->rt); + pd->rt.fv = nil; + } + if(pd->wt.fv) { + runtime·deltimer(&pd->wt); + pd->wt.fv = nil; + } + // Setup new timers. + if(d != 0 && d <= runtime·nanotime()) { + d = -1; + } + if(mode == 'r' || mode == 'r'+'w') + pd->rd = d; + if(mode == 'w' || mode == 'r'+'w') + pd->wd = d; + if(pd->rd > 0 && pd->rd == pd->wd) { + pd->rt.fv = &deadlineFn; + pd->rt.when = pd->rd; + // Copy current seq into the timer arg. + // Timer func will check the seq against current descriptor seq, + // if they differ the descriptor was reused or timers were reset. + pd->rt.arg.type = (Type*)pd->seq; + pd->rt.arg.data = pd; + runtime·addtimer(&pd->rt); + } else { + if(pd->rd > 0) { + pd->rt.fv = &readDeadlineFn; + pd->rt.when = pd->rd; + pd->rt.arg.type = (Type*)pd->seq; + pd->rt.arg.data = pd; + runtime·addtimer(&pd->rt); + } + if(pd->wd > 0) { + pd->wt.fv = &writeDeadlineFn; + pd->wt.when = pd->wd; + pd->wt.arg.type = (Type*)pd->seq; + pd->wt.arg.data = pd; + runtime·addtimer(&pd->wt); + } + } +ret: + runtime·unlock(pd); +} + +func runtime_pollUnblock(pd *PollDesc) { + G *rg, *wg; + + runtime·lock(pd); + if(pd->closing) + runtime·throw("runtime_pollUnblock: already closing"); + pd->closing = true; + pd->seq++; + rg = netpollunblock(pd, 'r'); + wg = netpollunblock(pd, 'w'); + if(pd->rt.fv) { + runtime·deltimer(&pd->rt); + pd->rt.fv = nil; + } + if(pd->wt.fv) { + runtime·deltimer(&pd->wt); + pd->wt.fv = nil; + } + runtime·unlock(pd); + if(rg) + runtime·ready(rg); + if(wg) + runtime·ready(wg); +} + +// make pd ready, newly runnable goroutines (if any) are enqueued info gpp list +void +runtime·netpollready(G **gpp, PollDesc *pd, int32 mode) +{ + G *rg, *wg; + + rg = wg = nil; + runtime·lock(pd); + if(mode == 'r' || mode == 'r'+'w') + rg = netpollunblock(pd, 'r'); + if(mode == 'w' || mode == 'r'+'w') + wg = netpollunblock(pd, 'w'); + runtime·unlock(pd); + if(rg) { + rg->schedlink = *gpp; + *gpp = rg; + } + if(wg) { + wg->schedlink = *gpp; + *gpp = wg; + } +} + +static intgo +checkerr(PollDesc *pd, int32 mode) +{ + if(pd->closing) + return 1; // errClosing + if((mode == 'r' && pd->rd < 0) || (mode == 'w' && pd->wd < 0)) + return 2; // errTimeout + return 0; +} + +static void +netpollblock(PollDesc *pd, int32 mode) +{ + G **gpp; + + gpp = &pd->rg; + if(mode == 'w') + gpp = &pd->wg; + if(*gpp == READY) { + *gpp = nil; + return; + } + if(*gpp != nil) + runtime·throw("epoll: double wait"); + *gpp = g; + runtime·park(runtime·unlock, &pd->Lock, "IO wait"); + runtime·lock(pd); +} + +static G* +netpollunblock(PollDesc *pd, int32 mode) +{ + G **gpp, *old; + + gpp = &pd->rg; + if(mode == 'w') + gpp = &pd->wg; + if(*gpp == READY) + return nil; + if(*gpp == nil) { + *gpp = READY; + return nil; + } + old = *gpp; + *gpp = nil; + return old; +} + +static void +deadlineimpl(int64 now, Eface arg, bool read, bool write) +{ + PollDesc *pd; + uint32 seq; + G *rg, *wg; + + USED(now); + pd = (PollDesc*)arg.data; + // This is the seq when the timer was set. + // If it's stale, ignore the timer event. + seq = (uintptr)arg.type; + rg = wg = nil; + runtime·lock(pd); + if(seq != pd->seq) { + // The descriptor was reused or timers were reset. + runtime·unlock(pd); + return; + } + if(read) { + if(pd->rd <= 0 || pd->rt.fv == nil) + runtime·throw("deadlineimpl: inconsistent read deadline"); + pd->rd = -1; + pd->rt.fv = nil; + rg = netpollunblock(pd, 'r'); + } + if(write) { + if(pd->wd <= 0 || (pd->wt.fv == nil && !read)) + runtime·throw("deadlineimpl: inconsistent write deadline"); + pd->wd = -1; + pd->wt.fv = nil; + wg = netpollunblock(pd, 'w'); + } + runtime·unlock(pd); + if(rg) + runtime·ready(rg); + if(wg) + runtime·ready(wg); +} + +static void +deadline(int64 now, Eface arg) +{ + deadlineimpl(now, arg, true, true); +} + +static void +readDeadline(int64 now, Eface arg) +{ + deadlineimpl(now, arg, true, false); +} + +static void +writeDeadline(int64 now, Eface arg) +{ + deadlineimpl(now, arg, false, true); +} + +static PollDesc* +allocPollDesc(void) +{ + PollDesc *pd; + uint32 i, n; + + runtime·lock(&pollcache); + if(pollcache.first == nil) { + n = PageSize/sizeof(*pd); + if(n == 0) + n = 1; + // Must be in non-GC memory because can be referenced + // only from epoll/kqueue internals. + pd = runtime·SysAlloc(n*sizeof(*pd)); + for(i = 0; i < n; i++) { + pd[i].link = pollcache.first; + pollcache.first = &pd[i]; + } + } + pd = pollcache.first; + pollcache.first = pd->link; + runtime·unlock(&pollcache); + return pd; +} diff --git a/src/pkg/runtime/netpoll_epoll.c b/src/pkg/runtime/netpoll_epoll.c new file mode 100644 index 000000000..d6ef0d144 --- /dev/null +++ b/src/pkg/runtime/netpoll_epoll.c @@ -0,0 +1,92 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" + +int32 runtime·epollcreate(int32 size); +int32 runtime·epollcreate1(int32 flags); +int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); +int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); +void runtime·closeonexec(int32 fd); + +static int32 epfd = -1; // epoll descriptor + +void +runtime·netpollinit(void) +{ + epfd = runtime·epollcreate1(EPOLL_CLOEXEC); + if(epfd >= 0) + return; + epfd = runtime·epollcreate(1024); + if(epfd >= 0) { + runtime·closeonexec(epfd); + return; + } + runtime·printf("netpollinit: failed to create descriptor (%d)\n", -epfd); + runtime·throw("netpollinit: failed to create descriptor"); +} + +int32 +runtime·netpollopen(int32 fd, PollDesc *pd) +{ + EpollEvent ev; + int32 res; + + ev.events = EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET; + ev.data = (uint64)pd; + res = runtime·epollctl(epfd, EPOLL_CTL_ADD, fd, &ev); + return -res; +} + +int32 +runtime·netpollclose(int32 fd) +{ + EpollEvent ev; + int32 res; + + res = runtime·epollctl(epfd, EPOLL_CTL_DEL, fd, &ev); + return -res; +} + +// polls for ready network connections +// returns list of goroutines that become runnable +G* +runtime·netpoll(bool block) +{ + EpollEvent events[128], *ev; + int32 n, i, waitms, mode; + G *gp; + + if(epfd == -1) + return nil; + waitms = -1; + if(!block) + waitms = 0; +retry: + n = runtime·epollwait(epfd, events, nelem(events), waitms); + if(n < 0) { + if(n != -EINTR) + runtime·printf("epollwait failed with %d\n", -n); + goto retry; + } + gp = nil; + for(i = 0; i < n; i++) { + ev = &events[i]; + if(ev->events == 0) + continue; + mode = 0; + if(ev->events & (EPOLLIN|EPOLLRDHUP|EPOLLHUP|EPOLLERR)) + mode += 'r'; + if(ev->events & (EPOLLOUT|EPOLLHUP|EPOLLERR)) + mode += 'w'; + if(mode) + runtime·netpollready(&gp, (void*)ev->data, mode); + } + if(block && gp == nil) + goto retry; + return gp; +} diff --git a/src/pkg/runtime/netpoll_kqueue.c b/src/pkg/runtime/netpoll_kqueue.c new file mode 100644 index 000000000..ad721e293 --- /dev/null +++ b/src/pkg/runtime/netpoll_kqueue.c @@ -0,0 +1,105 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" + +// Integrated network poller (kqueue-based implementation). + +int32 runtime·kqueue(void); +int32 runtime·kevent(int32, Kevent*, int32, Kevent*, int32, Timespec*); +void runtime·closeonexec(int32); + +static int32 kq = -1; + +void +runtime·netpollinit(void) +{ + kq = runtime·kqueue(); + if(kq < 0) { + runtime·printf("netpollinit: kqueue failed with %d\n", -kq); + runtime·throw("netpollinit: kqueue failed"); + } + runtime·closeonexec(kq); +} + +int32 +runtime·netpollopen(int32 fd, PollDesc *pd) +{ + Kevent ev[2]; + int32 n; + + // Arm both EVFILT_READ and EVFILT_WRITE in edge-triggered mode (EV_CLEAR) + // for the whole fd lifetime. The notifications are automatically unregistered + // when fd is closed. + ev[0].ident = fd; + ev[0].filter = EVFILT_READ; + ev[0].flags = EV_ADD|EV_RECEIPT|EV_CLEAR; + ev[0].fflags = 0; + ev[0].data = 0; + ev[0].udata = (byte*)pd; + ev[1] = ev[0]; + ev[1].filter = EVFILT_WRITE; + n = runtime·kevent(kq, ev, 2, ev, 2, nil); + if(n < 0) + return -n; + if(n != 2 || + (ev[0].flags&EV_ERROR) == 0 || ev[0].ident != fd || ev[0].filter != EVFILT_READ || + (ev[1].flags&EV_ERROR) == 0 || ev[1].ident != fd || ev[1].filter != EVFILT_WRITE) + return EFAULT; // just to mark out from other errors + if(ev[0].data != 0) + return ev[0].data; + if(ev[1].data != 0) + return ev[1].data; + return 0; +} + +int32 +runtime·netpollclose(int32 fd) +{ + // Don't need to unregister because calling close() + // on fd will remove any kevents that reference the descriptor. + USED(fd); + return 0; +} + +// Polls for ready network connections. +// Returns list of goroutines that become runnable. +G* +runtime·netpoll(bool block) +{ + Kevent events[64], *ev; + Timespec ts, *tp; + int32 n, i; + G *gp; + + if(kq == -1) + return nil; + tp = nil; + if(!block) { + ts.tv_sec = 0; + ts.tv_nsec = 0; + tp = &ts; + } + gp = nil; +retry: + n = runtime·kevent(kq, nil, 0, events, nelem(events), tp); + if(n < 0) { + if(n != -EINTR) + runtime·printf("kqueue failed with %d\n", -n); + goto retry; + } + for(i = 0; i < n; i++) { + ev = &events[i]; + if(ev->filter == EVFILT_READ) + runtime·netpollready(&gp, (PollDesc*)ev->udata, 'r'); + if(ev->filter == EVFILT_WRITE) + runtime·netpollready(&gp, (PollDesc*)ev->udata, 'w'); + } + if(block && gp == nil) + goto retry; + return gp; +} diff --git a/src/pkg/runtime/netpoll_stub.c b/src/pkg/runtime/netpoll_stub.c new file mode 100644 index 000000000..39d19a4ce --- /dev/null +++ b/src/pkg/runtime/netpoll_stub.c @@ -0,0 +1,18 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd netbsd openbsd plan9 windows + +#include "runtime.h" + +// Polls for ready network connections. +// Returns list of goroutines that become runnable. +G* +runtime·netpoll(bool block) +{ + // Implementation for platforms that do not support + // integrated network poller. + USED(block); + return nil; +} diff --git a/src/pkg/runtime/thread_darwin.c b/src/pkg/runtime/os_darwin.c index adb1ffe6a..6216e3a3c 100644 --- a/src/pkg/runtime/thread_darwin.c +++ b/src/pkg/runtime/os_darwin.c @@ -5,6 +5,7 @@ #include "runtime.h" #include "defs_GOOS_GOARCH.h" #include "os_GOOS.h" +#include "signal_unix.h" #include "stack.h" extern SigTab runtime·sigtab[]; @@ -69,6 +70,22 @@ runtime·osinit(void) } void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + static byte urandom_data[HashRandomBytes]; + int32 fd; + fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); + if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { + *rnd = urandom_data; + *rnd_len = HashRandomBytes; + } else { + *rnd = nil; + *rnd_len = 0; + } + runtime·close(fd); +} + +void runtime·goenvs(void) { runtime·goenvs_unix(); @@ -392,9 +409,14 @@ int32 runtime·mach_semacquire(uint32 sem, int64 ns) { int32 r; + int64 secs; if(ns >= 0) { - r = runtime·mach_semaphore_timedwait(sem, ns/1000000000LL, ns%1000000000LL); + secs = ns/1000000000LL; + // Avoid overflow + if(secs > 1LL<<30) + secs = 1LL<<30; + r = runtime·mach_semaphore_timedwait(sem, secs, ns%1000000000LL); if(r == KERN_ABORTED || r == KERN_OPERATION_TIMED_OUT) return -1; if(r != 0) @@ -530,3 +552,41 @@ runtime·badsignal(int32 sig) runtime·write(2, "\n", 1); runtime·exit(1); } + +void +runtime·setsig(int32 i, GoSighandler *fn, bool restart) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + sa.sa_flags = SA_SIGINFO|SA_ONSTACK; + if(restart) + sa.sa_flags |= SA_RESTART; + sa.sa_mask = ~(uintptr)0; + sa.sa_tramp = (void*)runtime·sigtramp; // runtime·sigtramp's job is to call into real handler + *(uintptr*)sa.__sigaction_u = (uintptr)fn; + runtime·sigaction(i, &sa, nil); +} + +GoSighandler* +runtime·getsig(int32 i) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + runtime·sigaction(i, nil, &sa); + return *(void**)sa.__sigaction_u; +} + +void +runtime·signalstack(byte *p, int32 n) +{ + StackT st; + + st.ss_sp = (void*)p; + st.ss_size = n; + st.ss_flags = 0; + if(p == nil) + st.ss_flags = SS_DISABLE; + runtime·sigaltstack(&st, nil); +} diff --git a/src/pkg/runtime/os_darwin.h b/src/pkg/runtime/os_darwin.h index 5fcb717cb..802410975 100644 --- a/src/pkg/runtime/os_darwin.h +++ b/src/pkg/runtime/os_darwin.h @@ -2,9 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#define SIG_DFL ((void*)0) -#define SIG_IGN ((void*)1) -#define SIGHUP 1 #define SS_DISABLE 4 int32 runtime·bsdthread_create(void*, M*, G*, void(*)(void)); @@ -27,8 +24,6 @@ void runtime·sigprocmask(int32, Sigset*, Sigset*); struct Sigaction; void runtime·sigaction(uintptr, struct Sigaction*, struct Sigaction*); -void runtime·setsig(int32, void(*)(int32, Siginfo*, void*, G*), bool); -void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); struct StackT; void runtime·sigaltstack(struct StackT*, struct StackT*); @@ -36,7 +31,6 @@ void runtime·sigtramp(void); void runtime·sigpanic(void); void runtime·setitimer(int32, Itimerval*, Itimerval*); -void runtime·raisesigpipe(void); #define NSIG 32 #define SI_USER 0 /* empirically true, but not what headers say */ diff --git a/src/pkg/runtime/thread_freebsd.c b/src/pkg/runtime/os_freebsd.c index 3ae14ee0a..68c0f4750 100644 --- a/src/pkg/runtime/thread_freebsd.c +++ b/src/pkg/runtime/os_freebsd.c @@ -1,9 +1,11 @@ -// Use of this source file is governed by a BSD-style -// license that can be found in the LICENSE file.` +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. #include "runtime.h" #include "defs_GOOS_GOARCH.h" #include "os_GOOS.h" +#include "signal_unix.h" #include "stack.h" extern SigTab runtime·sigtab[]; @@ -44,11 +46,16 @@ runtime·futexsleep(uint32 *addr, uint32 val, int64 ns) { int32 ret; Timespec ts, *tsp; + int64 secs; if(ns < 0) tsp = nil; else { - ts.tv_sec = ns / 1000000000LL; + secs = ns / 1000000000LL; + // Avoid overflow + if(secs > 1LL<<30) + secs = 1LL<<30; + ts.tv_sec = secs; ts.tv_nsec = ns % 1000000000LL; tsp = &ts; } @@ -116,6 +123,22 @@ runtime·osinit(void) } void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + static byte urandom_data[HashRandomBytes]; + int32 fd; + fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); + if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { + *rnd = urandom_data; + *rnd_len = HashRandomBytes; + } else { + *rnd = nil; + *rnd_len = 0; + } + runtime·close(fd); +} + +void runtime·goenvs(void) { runtime·goenvs_unix(); @@ -241,3 +264,58 @@ runtime·badsignal(int32 sig) runtime·write(2, "\n", 1); runtime·exit(1); } + +extern void runtime·sigtramp(void); + +typedef struct sigaction { + union { + void (*__sa_handler)(int32); + void (*__sa_sigaction)(int32, Siginfo*, void *); + } __sigaction_u; /* signal handler */ + int32 sa_flags; /* see signal options below */ + Sigset sa_mask; /* signal mask to apply */ +} Sigaction; + +void +runtime·setsig(int32 i, GoSighandler *fn, bool restart) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + sa.sa_flags = SA_SIGINFO|SA_ONSTACK; + if(restart) + sa.sa_flags |= SA_RESTART; + sa.sa_mask.__bits[0] = ~(uint32)0; + sa.sa_mask.__bits[1] = ~(uint32)0; + sa.sa_mask.__bits[2] = ~(uint32)0; + sa.sa_mask.__bits[3] = ~(uint32)0; + if(fn == runtime·sighandler) + fn = (void*)runtime·sigtramp; + sa.__sigaction_u.__sa_sigaction = (void*)fn; + runtime·sigaction(i, &sa, nil); +} + +GoSighandler* +runtime·getsig(int32 i) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + runtime·sigaction(i, nil, &sa); + if((void*)sa.__sigaction_u.__sa_sigaction == runtime·sigtramp) + return runtime·sighandler; + return (void*)sa.__sigaction_u.__sa_sigaction; +} + +void +runtime·signalstack(byte *p, int32 n) +{ + StackT st; + + st.ss_sp = (void*)p; + st.ss_size = n; + st.ss_flags = 0; + if(p == nil) + st.ss_flags = SS_DISABLE; + runtime·sigaltstack(&st, nil); +} diff --git a/src/pkg/runtime/os_freebsd.h b/src/pkg/runtime/os_freebsd.h index a37ad7cd8..e9be1362c 100644 --- a/src/pkg/runtime/os_freebsd.h +++ b/src/pkg/runtime/os_freebsd.h @@ -1,20 +1,18 @@ -#define SIG_DFL ((void*)0) -#define SIG_IGN ((void*)1) -#define SIGHUP 1 +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + #define SS_DISABLE 4 int32 runtime·thr_new(ThrParam*, int32); -void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); void runtime·sigpanic(void); void runtime·sigaltstack(Sigaltstack*, Sigaltstack*); struct sigaction; void runtime·sigaction(int32, struct sigaction*, struct sigaction*); void runtime·sigprocmask(Sigset *, Sigset *); -void runtime·setsig(int32, void(*)(int32, Siginfo*, void*, G*), bool); void runtime·setitimer(int32, Itimerval*, Itimerval*); int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); -void runtime·raisesigpipe(void); #define NSIG 33 #define SI_USER 0x10001 diff --git a/src/pkg/runtime/os_freebsd_arm.c b/src/pkg/runtime/os_freebsd_arm.c new file mode 100644 index 000000000..7eaa45c44 --- /dev/null +++ b/src/pkg/runtime/os_freebsd_arm.c @@ -0,0 +1,23 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" + +void +runtime·checkgoarm(void) +{ + // TODO(minux) +} + +#pragma textflag 7 +int64 +runtime·cputicks(void) +{ + // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). + // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + // TODO: need more entropy to better seed fastrand1. + return runtime·nanotime(); +} diff --git a/src/pkg/runtime/thread_linux.c b/src/pkg/runtime/os_linux.c index 78ddef878..e4ae1a5d8 100644 --- a/src/pkg/runtime/thread_linux.c +++ b/src/pkg/runtime/os_linux.c @@ -5,14 +5,11 @@ #include "runtime.h" #include "defs_GOOS_GOARCH.h" #include "os_GOOS.h" +#include "signal_unix.h" #include "stack.h" extern SigTab runtime·sigtab[]; -int32 runtime·open(uint8*, int32, int32); -int32 runtime·close(int32); -int32 runtime·read(int32, void*, int32); - static Sigset sigset_none; static Sigset sigset_all = { ~(uint32)0, ~(uint32)0 }; @@ -29,9 +26,6 @@ enum { FUTEX_WAIT = 0, FUTEX_WAKE = 1, - - EINTR = 4, - EAGAIN = 11, }; // Atomically, @@ -42,15 +36,17 @@ void runtime·futexsleep(uint32 *addr, uint32 val, int64 ns) { Timespec ts, *tsp; + int64 secs; if(ns < 0) tsp = nil; else { - ts.tv_sec = ns/1000000000LL; - ts.tv_nsec = ns%1000000000LL; + secs = ns/1000000000LL; // Avoid overflow - if(ts.tv_sec > 1<<30) - ts.tv_sec = 1<<30; + if(secs > 1LL<<30) + secs = 1LL<<30; + ts.tv_sec = secs; + ts.tv_nsec = ns%1000000000LL; tsp = &ts; } @@ -164,6 +160,32 @@ runtime·osinit(void) runtime·ncpu = getproccount(); } +// Random bytes initialized at startup. These come +// from the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.c). +byte* runtime·startup_random_data; +uint32 runtime·startup_random_data_len; + +void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + if(runtime·startup_random_data != nil) { + *rnd = runtime·startup_random_data; + *rnd_len = runtime·startup_random_data_len; + } else { + static byte urandom_data[HashRandomBytes]; + int32 fd; + fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); + if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { + *rnd = urandom_data; + *rnd_len = HashRandomBytes; + } else { + *rnd = nil; + *rnd_len = 0; + } + runtime·close(fd); + } +} + void runtime·goenvs(void) { @@ -203,8 +225,8 @@ runtime·sigpanic(void) if(g->sigcode0 == BUS_ADRERR && g->sigcode1 < 0x1000) { if(g->sigpc == 0) runtime·panicstring("call of nil func value"); - } runtime·panicstring("invalid memory address or nil pointer dereference"); + } runtime·printf("unexpected fault address %p\n", g->sigcode1); runtime·throw("fault"); case SIGSEGV: @@ -290,3 +312,60 @@ runtime·badsignal(int32 sig) runtime·write(2, "\n", 1); runtime·exit(1); } + +#ifdef GOARCH_386 +#define sa_handler k_sa_handler +#endif + +/* + * This assembler routine takes the args from registers, puts them on the stack, + * and calls sighandler(). + */ +extern void runtime·sigtramp(void); +extern void runtime·sigreturn(void); // calls runtime·sigreturn + +void +runtime·setsig(int32 i, GoSighandler *fn, bool restart) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + sa.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTORER; + if(restart) + sa.sa_flags |= SA_RESTART; + sa.sa_mask = ~0ULL; + // TODO(adonovan): Linux manpage says "sa_restorer element is + // obsolete and should not be used". Avoid it here, and test. + sa.sa_restorer = (void*)runtime·sigreturn; + if(fn == runtime·sighandler) + fn = (void*)runtime·sigtramp; + sa.sa_handler = fn; + if(runtime·rt_sigaction(i, &sa, nil, sizeof(sa.sa_mask)) != 0) + runtime·throw("rt_sigaction failure"); +} + +GoSighandler* +runtime·getsig(int32 i) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + if(runtime·rt_sigaction(i, nil, &sa, sizeof(sa.sa_mask)) != 0) + runtime·throw("rt_sigaction read failure"); + if((void*)sa.sa_handler == runtime·sigtramp) + return runtime·sighandler; + return (void*)sa.sa_handler; +} + +void +runtime·signalstack(byte *p, int32 n) +{ + Sigaltstack st; + + st.ss_sp = p; + st.ss_size = n; + st.ss_flags = 0; + if(p == nil) + st.ss_flags = SS_DISABLE; + runtime·sigaltstack(&st, nil); +} diff --git a/src/pkg/runtime/os_linux.h b/src/pkg/runtime/os_linux.h index a23fe0f73..b2d3f6f2a 100644 --- a/src/pkg/runtime/os_linux.h +++ b/src/pkg/runtime/os_linux.h @@ -2,9 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#define SIG_DFL ((void*)0) -#define SIG_IGN ((void*)1) -#define SIGHUP 1 #define SS_DISABLE 2 // Linux-specific system calls @@ -13,14 +10,11 @@ int32 runtime·clone(int32, void*, M*, G*, void(*)(void)); struct Sigaction; int32 runtime·rt_sigaction(uintptr, struct Sigaction*, void*, uintptr); -void runtime·setsig(int32, void(*)(int32, Siginfo*, void*, G*), bool); -void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); void runtime·sigaltstack(Sigaltstack*, Sigaltstack*); void runtime·sigpanic(void); void runtime·setitimer(int32, Itimerval*, Itimerval*); -void runtime·raisesigpipe(void); #define NSIG 65 #define SI_USER 0 diff --git a/src/pkg/runtime/os_linux_386.c b/src/pkg/runtime/os_linux_386.c new file mode 100644 index 000000000..18becb6e6 --- /dev/null +++ b/src/pkg/runtime/os_linux_386.c @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" + +#define AT_NULL 0 +#define AT_RANDOM 25 +#define AT_SYSINFO 32 +extern uint32 runtime·_vdso; + +#pragma textflag 7 +void +runtime·linux_setup_vdso(int32 argc, byte **argv) +{ + byte **envp; + uint32 *auxv; + + // skip envp to get to ELF auxiliary vector. + for(envp = &argv[argc+1]; *envp != nil; envp++) + ; + envp++; + + for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { + if(auxv[0] == AT_SYSINFO) { + runtime·_vdso = auxv[1]; + continue; + } + if(auxv[0] == AT_RANDOM) { + runtime·startup_random_data = (byte*)auxv[1]; + runtime·startup_random_data_len = 16; + continue; + } + } +} diff --git a/src/pkg/runtime/os_linux_arm.c b/src/pkg/runtime/os_linux_arm.c new file mode 100644 index 000000000..dd0fa9415 --- /dev/null +++ b/src/pkg/runtime/os_linux_arm.c @@ -0,0 +1,82 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" + +#define AT_NULL 0 +#define AT_PLATFORM 15 // introduced in at least 2.6.11 +#define AT_HWCAP 16 // introduced in at least 2.6.11 +#define AT_RANDOM 25 // introduced in 2.6.29 +#define HWCAP_VFP (1 << 6) // introduced in at least 2.6.11 +#define HWCAP_VFPv3 (1 << 13) // introduced in 2.6.30 +static uint32 runtime·randomNumber; +uint8 runtime·armArch = 6; // we default to ARMv6 +uint32 runtime·hwcap; // set by setup_auxv +uint8 runtime·goarm; // set by 5l + +void +runtime·checkgoarm(void) +{ + if(runtime·goarm > 5 && !(runtime·hwcap & HWCAP_VFP)) { + runtime·printf("runtime: this CPU has no floating point hardware, so it cannot run\n"); + runtime·printf("this GOARM=%d binary. Recompile using GOARM=5.\n", runtime·goarm); + runtime·exit(1); + } + if(runtime·goarm > 6 && !(runtime·hwcap & HWCAP_VFPv3)) { + runtime·printf("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n"); + runtime·printf("this GOARM=%d binary. Recompile using GOARM=6.\n", runtime·goarm); + runtime·exit(1); + } +} + +#pragma textflag 7 +void +runtime·setup_auxv(int32 argc, void *argv_list) +{ + byte **argv; + byte **envp; + byte *rnd; + uint32 *auxv; + uint32 t; + + argv = &argv_list; + + // skip envp to get to ELF auxiliary vector. + for(envp = &argv[argc+1]; *envp != nil; envp++) + ; + envp++; + + for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { + switch(auxv[0]) { + case AT_RANDOM: // kernel provided 16-byte worth of random data + if(auxv[1]) { + rnd = (byte*)auxv[1]; + runtime·randomNumber = rnd[4] | rnd[5]<<8 | rnd[6]<<16 | rnd[7]<<24; + } + break; + case AT_PLATFORM: // v5l, v6l, v7l + if(auxv[1]) { + t = *(uint8*)(auxv[1]+1); + if(t >= '5' && t <= '7') + runtime·armArch = t - '0'; + } + break; + case AT_HWCAP: // CPU capability bit flags + runtime·hwcap = auxv[1]; + break; + } + } +} + +#pragma textflag 7 +int64 +runtime·cputicks(void) +{ + // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). + // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + // runtime·randomNumber provides better seeding of fastrand1. + return runtime·nanotime() + runtime·randomNumber; +} diff --git a/src/pkg/runtime/thread_netbsd.c b/src/pkg/runtime/os_netbsd.c index f333c6dd8..936334cac 100644 --- a/src/pkg/runtime/thread_netbsd.c +++ b/src/pkg/runtime/os_netbsd.c @@ -1,9 +1,11 @@ -// Use of this source file is governed by a BSD-style -// license that can be found in the LICENSE file.` +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. #include "runtime.h" #include "defs_GOOS_GOARCH.h" #include "os_GOOS.h" +#include "signal_unix.h" #include "stack.h" enum @@ -181,6 +183,22 @@ runtime·osinit(void) } void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + static byte urandom_data[HashRandomBytes]; + int32 fd; + fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); + if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { + *rnd = urandom_data; + *rnd_len = HashRandomBytes; + } else { + *rnd = nil; + *rnd_len = 0; + } + runtime·close(fd); +} + +void runtime·goenvs(void) { runtime·goenvs_unix(); @@ -286,3 +304,58 @@ runtime·badsignal(int32 sig) runtime·write(2, "\n", 1); runtime·exit(1); } + +extern void runtime·sigtramp(void); + +typedef struct sigaction { + union { + void (*_sa_handler)(int32); + void (*_sa_sigaction)(int32, Siginfo*, void *); + } _sa_u; /* signal handler */ + uint32 sa_mask[4]; /* signal mask to apply */ + int32 sa_flags; /* see signal options below */ +} Sigaction; + +void +runtime·setsig(int32 i, GoSighandler *fn, bool restart) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + sa.sa_flags = SA_SIGINFO|SA_ONSTACK; + if(restart) + sa.sa_flags |= SA_RESTART; + sa.sa_mask[0] = ~0U; + sa.sa_mask[1] = ~0U; + sa.sa_mask[2] = ~0U; + sa.sa_mask[3] = ~0U; + if (fn == runtime·sighandler) + fn = (void*)runtime·sigtramp; + sa._sa_u._sa_sigaction = (void*)fn; + runtime·sigaction(i, &sa, nil); +} + +GoSighandler* +runtime·getsig(int32 i) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + runtime·sigaction(i, nil, &sa); + if((void*)sa._sa_u._sa_sigaction == runtime·sigtramp) + return runtime·sighandler; + return (void*)sa._sa_u._sa_sigaction; +} + +void +runtime·signalstack(byte *p, int32 n) +{ + StackT st; + + st.ss_sp = (void*)p; + st.ss_size = n; + st.ss_flags = 0; + if(p == nil) + st.ss_flags = SS_DISABLE; + runtime·sigaltstack(&st, nil); +} diff --git a/src/pkg/runtime/os_netbsd.h b/src/pkg/runtime/os_netbsd.h index 19d72fd25..c193ae0b4 100644 --- a/src/pkg/runtime/os_netbsd.h +++ b/src/pkg/runtime/os_netbsd.h @@ -2,9 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#define SIG_DFL ((void*)0) -#define SIG_IGN ((void*)1) -#define SIGHUP 1 #define SS_DISABLE 4 #define SIG_BLOCK 1 @@ -13,9 +10,6 @@ struct sigaction; -void runtime·raisesigpipe(void); -void runtime·setsig(int32, void(*)(int32, Siginfo*, void*, G*), bool); -void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); void runtime·sigpanic(void); void runtime·setitimer(int32, Itimerval*, Itimerval*); @@ -23,6 +17,7 @@ void runtime·sigaction(int32, struct sigaction*, struct sigaction*); void runtime·sigaltstack(Sigaltstack*, Sigaltstack*); void runtime·sigprocmask(int32, Sigset*, Sigset*); int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); +extern void runtime·lwp_tramp(void); #define NSIG 33 #define SI_USER 0 diff --git a/src/pkg/runtime/os_netbsd_386.c b/src/pkg/runtime/os_netbsd_386.c new file mode 100644 index 000000000..23e9db3c1 --- /dev/null +++ b/src/pkg/runtime/os_netbsd_386.c @@ -0,0 +1,17 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" + +void +runtime·lwp_mcontext_init(McontextT *mc, void *stack, M *mp, G *gp, void (*fn)(void)) +{ + mc->__gregs[REG_EIP] = (uint32)runtime·lwp_tramp; + mc->__gregs[REG_UESP] = (uint32)stack; + mc->__gregs[REG_EBX] = (uint32)mp; + mc->__gregs[REG_EDX] = (uint32)gp; + mc->__gregs[REG_ESI] = (uint32)fn; +} diff --git a/src/pkg/runtime/os_netbsd_amd64.c b/src/pkg/runtime/os_netbsd_amd64.c new file mode 100644 index 000000000..226846cbb --- /dev/null +++ b/src/pkg/runtime/os_netbsd_amd64.c @@ -0,0 +1,18 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" + +void +runtime·lwp_mcontext_init(McontextT *mc, void *stack, M *mp, G *gp, void (*fn)(void)) +{ + // Machine dependent mcontext initialisation for LWP. + mc->__gregs[REG_RIP] = (uint64)runtime·lwp_tramp; + mc->__gregs[REG_RSP] = (uint64)stack; + mc->__gregs[REG_R8] = (uint64)mp; + mc->__gregs[REG_R9] = (uint64)gp; + mc->__gregs[REG_R12] = (uint64)fn; +} diff --git a/src/pkg/runtime/os_netbsd_arm.c b/src/pkg/runtime/os_netbsd_arm.c new file mode 100644 index 000000000..385e6406d --- /dev/null +++ b/src/pkg/runtime/os_netbsd_arm.c @@ -0,0 +1,33 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" +#include "signal_GOOS_GOARCH.h" + +void +runtime·lwp_mcontext_init(McontextT *mc, void *stack, M *mp, G *gp, void (*fn)(void)) +{ + mc->__gregs[REG_R15] = (uint32)runtime·lwp_tramp; + mc->__gregs[REG_R13] = (uint32)stack; + mc->__gregs[REG_R0] = (uint32)mp; + mc->__gregs[REG_R1] = (uint32)gp; + mc->__gregs[REG_R2] = (uint32)fn; +} + +void +runtime·checkgoarm(void) +{ + // TODO(minux) +} + +#pragma textflag 7 +int64 +runtime·cputicks() { + // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). + // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + // TODO: need more entropy to better seed fastrand1. + return runtime·nanotime(); +} diff --git a/src/pkg/runtime/thread_openbsd.c b/src/pkg/runtime/os_openbsd.c index 700c48147..4ce64f9f2 100644 --- a/src/pkg/runtime/thread_openbsd.c +++ b/src/pkg/runtime/os_openbsd.c @@ -1,9 +1,11 @@ -// Use of this source file is governed by a BSD-style -// license that can be found in the LICENSE file.` +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. #include "runtime.h" #include "defs_GOOS_GOARCH.h" #include "os_GOOS.h" +#include "signal_unix.h" #include "stack.h" enum @@ -61,6 +63,7 @@ int32 runtime·semasleep(int64 ns) { Timespec ts; + int64 secs; // spin-mutex lock while(runtime·xchg(&m->waitsemalock, 1)) @@ -75,7 +78,11 @@ runtime·semasleep(int64 ns) runtime·thrsleep(&m->waitsemacount, 0, nil, &m->waitsemalock, nil); else { ns += runtime·nanotime(); - ts.tv_sec = ns/1000000000LL; + secs = ns/1000000000LL; + // Avoid overflow + if(secs >= 1LL<<31) + secs = (1LL<<31) - 1; + ts.tv_sec = secs; ts.tv_nsec = ns%1000000000LL; runtime·thrsleep(&m->waitsemacount, CLOCK_REALTIME, &ts, &m->waitsemalock, nil); } @@ -160,6 +167,22 @@ runtime·osinit(void) } void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + static byte urandom_data[HashRandomBytes]; + int32 fd; + fd = runtime·open("/dev/urandom", 0 /* O_RDONLY */, 0); + if(runtime·read(fd, urandom_data, HashRandomBytes) == HashRandomBytes) { + *rnd = urandom_data; + *rnd_len = HashRandomBytes; + } else { + *rnd = nil; + *rnd_len = 0; + } + runtime·close(fd); +} + +void runtime·goenvs(void) { runtime·goenvs_unix(); @@ -263,3 +286,55 @@ runtime·badsignal(int32 sig) runtime·write(2, "\n", 1); runtime·exit(1); } + +extern void runtime·sigtramp(void); + +typedef struct sigaction { + union { + void (*__sa_handler)(int32); + void (*__sa_sigaction)(int32, Siginfo*, void *); + } __sigaction_u; /* signal handler */ + uint32 sa_mask; /* signal mask to apply */ + int32 sa_flags; /* see signal options below */ +} Sigaction; + +void +runtime·setsig(int32 i, GoSighandler *fn, bool restart) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + sa.sa_flags = SA_SIGINFO|SA_ONSTACK; + if(restart) + sa.sa_flags |= SA_RESTART; + sa.sa_mask = ~0U; + if(fn == runtime·sighandler) + fn = (void*)runtime·sigtramp; + sa.__sigaction_u.__sa_sigaction = (void*)fn; + runtime·sigaction(i, &sa, nil); +} + +GoSighandler* +runtime·getsig(int32 i) +{ + Sigaction sa; + + runtime·memclr((byte*)&sa, sizeof sa); + runtime·sigaction(i, nil, &sa); + if((void*)sa.__sigaction_u.__sa_sigaction == runtime·sigtramp) + return runtime·sighandler; + return (void*)sa.__sigaction_u.__sa_sigaction; +} + +void +runtime·signalstack(byte *p, int32 n) +{ + StackT st; + + st.ss_sp = (void*)p; + st.ss_size = n; + st.ss_flags = 0; + if(p == nil) + st.ss_flags = SS_DISABLE; + runtime·sigaltstack(&st, nil); +} diff --git a/src/pkg/runtime/os_openbsd.h b/src/pkg/runtime/os_openbsd.h index a599aad05..dbfa4b69f 100644 --- a/src/pkg/runtime/os_openbsd.h +++ b/src/pkg/runtime/os_openbsd.h @@ -2,9 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#define SIG_DFL ((void*)0) -#define SIG_IGN ((void*)1) -#define SIGHUP 1 #define SS_DISABLE 4 #define SIG_BLOCK 1 @@ -13,14 +10,11 @@ struct sigaction; -void runtime·raisesigpipe(void); -void runtime·setsig(int32, void(*)(int32, Siginfo*, void*, G*), bool); void runtime·sigpanic(void); void runtime·setitimer(int32, Itimerval*, Itimerval*); void runtime·sigaction(int32, struct sigaction*, struct sigaction*); void runtime·sigaltstack(Sigaltstack*, Sigaltstack*); -void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); Sigset runtime·sigprocmask(int32, Sigset); int32 runtime·sysctl(uint32*, uint32, byte*, uintptr*, byte*, uintptr); diff --git a/src/pkg/runtime/thread_plan9.c b/src/pkg/runtime/os_plan9.c index 7f94623e7..c7ed59fc9 100644 --- a/src/pkg/runtime/thread_plan9.c +++ b/src/pkg/runtime/os_plan9.c @@ -19,6 +19,10 @@ runtime·mpreinit(M *mp) // Initialize stack and goroutine for note handling. mp->gsignal = runtime·malg(32*1024); mp->notesig = (int8*)runtime·malloc(ERRMAX*sizeof(int8)); + + // Initialize stack for handling strings from the + // errstr system call, as used in package syscall. + mp->errstr = (byte*)runtime·malloc(ERRMAX*sizeof(byte)); } // Called to initialize a new m (including the bootstrap m). @@ -44,7 +48,7 @@ getproccount(void) int32 fd, i, n, ncpu; byte buf[2048]; - fd = runtime·open((byte*)"/dev/sysstat", OREAD); + fd = runtime·open("/dev/sysstat", OREAD, 0); if(fd < 0) return 1; ncpu = 0; @@ -68,7 +72,7 @@ getpid(void) int32 fd; runtime·memclr(b, sizeof(b)); - fd = runtime·open((byte*)"#c/pid", 0); + fd = runtime·open("#c/pid", 0, 0); if(fd >= 0) { runtime·read(fd, b, sizeof(b)); runtime·close(fd); @@ -88,6 +92,20 @@ runtime·osinit(void) } void +runtime·crash(void) +{ + runtime·notify(nil); + *(int32*)0 = 0; +} + +void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + *rnd = nil; + *rnd_len = 0; +} + +void runtime·goenvs(void) { } @@ -191,7 +209,7 @@ runtime·postnote(int32 pid, int8* msg) p--; runtime·memmove((void*)p, (void*)"/note", 5); - fd = runtime·open(buf, OWRITE); + fd = runtime·open((int8*)buf, OWRITE, 0); if(fd < 0) return -1; diff --git a/src/pkg/runtime/os_plan9.h b/src/pkg/runtime/os_plan9.h index c2cdf5b44..f0474cda5 100644 --- a/src/pkg/runtime/os_plan9.h +++ b/src/pkg/runtime/os_plan9.h @@ -3,12 +3,9 @@ // license that can be found in the LICENSE file. // Plan 9-specific system calls -int32 runtime·open(uint8 *file, int32 mode); int32 runtime·pread(int32 fd, void *buf, int32 nbytes, int64 offset); int32 runtime·pwrite(int32 fd, void *buf, int32 nbytes, int64 offset); -int32 runtime·read(int32 fd, void *buf, int32 nbytes); int64 runtime·seek(int32 fd, int64 offset, int32 whence); -int32 runtime·close(int32 fd); void runtime·exits(int8* msg); intptr runtime·brk_(void*); int32 runtime·sleep(int32 ms); @@ -19,7 +16,6 @@ int32 runtime·plan9_semrelease(uint32 *addr, int32 count); int32 runtime·notify(void (*fn)(void*, int8*)); int32 runtime·noted(int32); void runtime·sigtramp(void*, int8*); -int32 runtime·sighandler(void*, int8*, G*); void runtime·sigpanic(void); void runtime·goexitsall(int8*); void runtime·setfpmasks(void); diff --git a/src/pkg/runtime/signal_plan9_386.c b/src/pkg/runtime/os_plan9_386.c index 17bc11749..3396e44e7 100644 --- a/src/pkg/runtime/signal_plan9_386.c +++ b/src/pkg/runtime/os_plan9_386.c @@ -28,6 +28,7 @@ runtime·dumpregs(Ureg *u) int32 runtime·sighandler(void *v, int8 *s, G *gp) { + bool crash; Ureg *ureg; uintptr *sp; SigTab *sig, *nsig; @@ -93,11 +94,15 @@ Throw: runtime·printf("PC=%X\n", ureg->pc); runtime·printf("\n"); - if(runtime·gotraceback()) { + if(runtime·gotraceback(&crash)) { runtime·traceback((void*)ureg->pc, (void*)ureg->sp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(ureg); } + + if(crash) + runtime·crash(); + runtime·goexitsall(""); runtime·exits(s); @@ -112,6 +117,12 @@ runtime·sigenable(uint32 sig) } void +runtime·sigdisable(uint32 sig) +{ + USED(sig); +} + +void runtime·resetcpuprofiler(int32 hz) { // TODO: Enable profiling interrupts. diff --git a/src/pkg/runtime/signal_plan9_amd64.c b/src/pkg/runtime/os_plan9_amd64.c index e4f946abc..cf0a82b6b 100644 --- a/src/pkg/runtime/signal_plan9_amd64.c +++ b/src/pkg/runtime/os_plan9_amd64.c @@ -36,6 +36,7 @@ runtime·dumpregs(Ureg *u) int32 runtime·sighandler(void *v, int8 *s, G *gp) { + bool crash; Ureg *ureg; uintptr *sp; SigTab *sig, *nsig; @@ -101,11 +102,15 @@ Throw: runtime·printf("PC=%X\n", ureg->ip); runtime·printf("\n"); - if(runtime·gotraceback()) { + if(runtime·gotraceback(&crash)) { runtime·traceback((void*)ureg->ip, (void*)ureg->sp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(ureg); } + + if(crash) + runtime·crash(); + runtime·goexitsall(""); runtime·exits(s); @@ -119,6 +124,12 @@ runtime·sigenable(uint32 sig) } void +runtime·sigdisable(uint32 sig) +{ + USED(sig); +} + +void runtime·resetcpuprofiler(int32 hz) { // TODO: Enable profiling interrupts. diff --git a/src/pkg/runtime/thread_windows.c b/src/pkg/runtime/os_windows.c index ae4e82e50..b28affe31 100644 --- a/src/pkg/runtime/thread_windows.c +++ b/src/pkg/runtime/os_windows.c @@ -11,6 +11,9 @@ #pragma dynimport runtime·CreateEvent CreateEventA "kernel32.dll" #pragma dynimport runtime·CreateThread CreateThread "kernel32.dll" #pragma dynimport runtime·CreateWaitableTimer CreateWaitableTimerA "kernel32.dll" +#pragma dynimport runtime·CryptAcquireContextW CryptAcquireContextW "advapi32.dll" +#pragma dynimport runtime·CryptGenRandom CryptGenRandom "advapi32.dll" +#pragma dynimport runtime·CryptReleaseContext CryptReleaseContext "advapi32.dll" #pragma dynimport runtime·DuplicateHandle DuplicateHandle "kernel32.dll" #pragma dynimport runtime·ExitProcess ExitProcess "kernel32.dll" #pragma dynimport runtime·FreeEnvironmentStringsW FreeEnvironmentStringsW "kernel32.dll" @@ -31,11 +34,17 @@ #pragma dynimport runtime·timeBeginPeriod timeBeginPeriod "winmm.dll" #pragma dynimport runtime·WaitForSingleObject WaitForSingleObject "kernel32.dll" #pragma dynimport runtime·WriteFile WriteFile "kernel32.dll" +#pragma dynimport runtime·NtWaitForSingleObject NtWaitForSingleObject "ntdll.dll" + +extern void *runtime·NtWaitForSingleObject; extern void *runtime·CloseHandle; extern void *runtime·CreateEvent; extern void *runtime·CreateThread; extern void *runtime·CreateWaitableTimer; +extern void *runtime·CryptAcquireContextW; +extern void *runtime·CryptGenRandom; +extern void *runtime·CryptReleaseContext; extern void *runtime·DuplicateHandle; extern void *runtime·ExitProcess; extern void *runtime·FreeEnvironmentStringsW; @@ -79,6 +88,24 @@ runtime·osinit(void) } void +runtime·get_random_data(byte **rnd, int32 *rnd_len) +{ + uintptr handle; + *rnd = nil; + *rnd_len = 0; + if(runtime·stdcall(runtime·CryptAcquireContextW, 5, &handle, nil, nil, + (uintptr)1 /* PROV_RSA_FULL */, + (uintptr)0xf0000000U /* CRYPT_VERIFYCONTEXT */) != 0) { + static byte random_data[HashRandomBytes]; + if(runtime·stdcall(runtime·CryptGenRandom, 3, handle, (uintptr)HashRandomBytes, random_data)) { + *rnd = random_data; + *rnd_len = HashRandomBytes; + } + runtime·stdcall(runtime·CryptReleaseContext, 2, handle, (uintptr)0); + } +} + +void runtime·goenvs(void) { extern Slice syscall·envs; @@ -135,22 +162,6 @@ runtime·write(int32 fd, void *buf, int32 n) return written; } -#pragma textflag 7 -void -runtime·osyield(void) -{ - runtime·stdcall(runtime·Sleep, 1, (uintptr)0); -} - -void -runtime·usleep(uint32 us) -{ - us /= 1000; - if(us == 0) - us = 1; - runtime·stdcall(runtime·Sleep, 1, (uintptr)us); -} - #define INFINITE ((uintptr)0xFFFFFFFF) int32 @@ -444,3 +455,16 @@ int32 runtime·badcallbacklen = sizeof runtime·badcallbackmsg - 1; int8 runtime·badsignalmsg[] = "runtime: signal received on thread not created by Go.\n"; int32 runtime·badsignallen = sizeof runtime·badsignalmsg - 1; + +void +runtime·crash(void) +{ + // TODO: This routine should do whatever is needed + // to make the Windows program abort/crash as it + // would if Go was not intercepting signals. + // On Unix the routine would remove the custom signal + // handler and then raise a signal (like SIGABRT). + // Something like that should happen here. + // It's okay to leave this empty for now: if crash returns + // the ordinary exit-after-panic happens. +} diff --git a/src/pkg/runtime/signal_windows_386.c b/src/pkg/runtime/os_windows_386.c index d76d5bf4b..20fbea13d 100644 --- a/src/pkg/runtime/signal_windows_386.c +++ b/src/pkg/runtime/os_windows_386.c @@ -27,6 +27,7 @@ runtime·dumpregs(Context *r) uint32 runtime·sighandler(ExceptionRecord *info, Context *r, G *gp) { + bool crash; uintptr *sp; switch(info->ExceptionCode) { @@ -74,11 +75,15 @@ runtime·sighandler(ExceptionRecord *info, Context *r, G *gp) } runtime·printf("\n"); - if(runtime·gotraceback()){ + if(runtime·gotraceback(&crash)){ runtime·traceback((void*)r->Eip, (void*)r->Esp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(r); } + + if(crash) + runtime·crash(); + runtime·exit(2); return 0; @@ -91,6 +96,12 @@ runtime·sigenable(uint32 sig) } void +runtime·sigdisable(uint32 sig) +{ + USED(sig); +} + +void runtime·dosigprof(Context *r, G *gp) { runtime·sigprof((uint8*)r->Eip, (uint8*)r->Esp, nil, gp); diff --git a/src/pkg/runtime/signal_windows_amd64.c b/src/pkg/runtime/os_windows_amd64.c index 3729aa57b..881c73c93 100644 --- a/src/pkg/runtime/signal_windows_amd64.c +++ b/src/pkg/runtime/os_windows_amd64.c @@ -35,6 +35,7 @@ runtime·dumpregs(Context *r) uint32 runtime·sighandler(ExceptionRecord *info, Context *r, G *gp) { + bool crash; uintptr *sp; switch(info->ExceptionCode) { @@ -81,11 +82,14 @@ runtime·sighandler(ExceptionRecord *info, Context *r, G *gp) } runtime·printf("\n"); - if(runtime·gotraceback()){ + if(runtime·gotraceback(&crash)){ runtime·traceback((void*)r->Rip, (void*)r->Rsp, 0, gp); runtime·tracebackothers(gp); runtime·dumpregs(r); } + + if(crash) + runtime·crash(); runtime·exit(2); return 0; @@ -98,6 +102,12 @@ runtime·sigenable(uint32 sig) } void +runtime·sigdisable(uint32 sig) +{ + USED(sig); +} + +void runtime·dosigprof(Context *r, G *gp) { runtime·sigprof((uint8*)r->Rip, (uint8*)r->Rsp, nil, gp); diff --git a/src/pkg/runtime/panic.c b/src/pkg/runtime/panic.c index 2f553f417..d0cf3ad6f 100644 --- a/src/pkg/runtime/panic.c +++ b/src/pkg/runtime/panic.c @@ -5,6 +5,7 @@ #include "runtime.h" #include "arch_GOARCH.h" #include "stack.h" +#include "malloc.h" // Code related to defer, panic and recover. @@ -383,7 +384,10 @@ nomatch: void runtime·startpanic(void) { - if(m->mcache == nil) // can happen if called from signal handler or throw + if(runtime·mheap == 0 || runtime·mheap->cachealloc.size == 0) { // very early + runtime·printf("runtime: panic before malloc heap initialized\n"); + m->mallocing = 1; // tell rest of panic not to try to malloc + } else if(m->mcache == nil) // can happen if called from signal handler or throw m->mcache = runtime·allocmcache(); if(m->dying) { runtime·printf("panic during panic\n"); @@ -398,12 +402,13 @@ void runtime·dopanic(int32 unused) { static bool didothers; + bool crash; if(g->sig != 0) runtime·printf("[signal %x code=%p addr=%p pc=%p]\n", g->sig, g->sigcode0, g->sigcode1, g->sigpc); - if(runtime·gotraceback()){ + if(runtime·gotraceback(&crash)){ if(g != m->g0) { runtime·printf("\n"); runtime·goroutineheader(g); @@ -424,6 +429,9 @@ runtime·dopanic(int32 unused) runtime·lock(&deadlock); runtime·lock(&deadlock); } + + if(crash) + runtime·crash(); runtime·exit(2); } diff --git a/src/pkg/runtime/parfor.c b/src/pkg/runtime/parfor.c index aa5537d02..a4468c2af 100644 --- a/src/pkg/runtime/parfor.c +++ b/src/pkg/runtime/parfor.c @@ -46,6 +46,7 @@ void runtime·parforsetup(ParFor *desc, uint32 nthr, uint32 n, void *ctx, bool wait, void (*body)(ParFor*, uint32)) { uint32 i, begin, end; + uint64 *pos; if(desc == nil || nthr == 0 || nthr > desc->nthrmax || body == nil) { runtime·printf("desc=%p nthr=%d count=%d body=%p\n", desc, nthr, n, body); @@ -67,7 +68,10 @@ runtime·parforsetup(ParFor *desc, uint32 nthr, uint32 n, void *ctx, bool wait, for(i=0; i<nthr; i++) { begin = (uint64)n*i / nthr; end = (uint64)n*(i+1) / nthr; - desc->thr[i].pos = (uint64)begin | (((uint64)end)<<32); + pos = &desc->thr[i].pos; + if(((uintptr)pos & 7) != 0) + runtime·throw("parforsetup: pos is not aligned"); + *pos = (uint64)begin | (((uint64)end)<<32); } } diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c index 4ce0a718c..8d05730e4 100644 --- a/src/pkg/runtime/proc.c +++ b/src/pkg/runtime/proc.c @@ -49,6 +49,7 @@ struct Sched { Note stopnote; uint32 sysmonwait; Note sysmonnote; + uint64 lastpoll; int32 profilehz; // cpu profiling rate }; @@ -71,8 +72,6 @@ M* runtime·extram; int8* runtime·goos; int32 runtime·ncpu; static int32 newprocs; -// Keep trace of scavenger's goroutine for deadlock detection. -static G *scvg; void runtime·mstart(void); static void runqput(P*, G*); @@ -109,6 +108,7 @@ static void globrunqput(G*); static G* globrunqget(P*); static P* pidleget(void); static void pidleput(P*); +static void injectglist(G*); // The bootstrap sequence is: // @@ -137,6 +137,7 @@ runtime·schedinit(void) // so that we don't need to call malloc when we crash. // runtime·findfunc(0); + runtime·sched.lastpoll = runtime·nanotime(); procs = 1; p = runtime·getenv("GOMAXPROCS"); if(p != nil && (n = runtime·atoi(p)) > 0) { @@ -174,8 +175,7 @@ runtime·main(void) runtime·lockOSThread(); if(m != &runtime·m0) runtime·throw("runtime·main not on m0"); - scvg = runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); - scvg->issystem = true; + runtime·newproc1(&scavenger, nil, 0, 0, runtime·main); main·init(); runtime·unlockOSThread(); @@ -232,7 +232,7 @@ runtime·tracebackothers(G *me) G *gp; int32 traceback; - traceback = runtime·gotraceback(); + traceback = runtime·gotraceback(nil); for(gp = runtime·allg; gp != nil; gp = gp->alllink) { if(gp == me || gp->status == Gdead) continue; @@ -332,7 +332,7 @@ runtime·helpgc(int32 nproc) mp = mget(); if(mp == nil) runtime·throw("runtime·gcprocs inconsistency"); - mp->helpgc = 1; + mp->helpgc = n; mp->mcache = runtime·allp[pos]->mcache; pos++; runtime·notewakeup(&mp->park); @@ -386,16 +386,19 @@ runtime·stoptheworld(void) static void mhelpgc(void) { - m->helpgc = 1; + m->helpgc = -1; } void runtime·starttheworld(void) { - P *p; + P *p, *p1; M *mp; + G *gp; bool add; + gp = runtime·netpoll(false); // non-blocking + injectglist(gp); add = needaddgcproc(); runtime·lock(&runtime·sched); if(newprocs) { @@ -405,6 +408,7 @@ runtime·starttheworld(void) procresize(runtime·gomaxprocs); runtime·gcwaiting = 0; + p1 = nil; while(p = pidleget()) { // procresize() puts p's with work at the beginning of the list. // Once we reach a p without a run queue, the rest don't have one either. @@ -414,8 +418,9 @@ runtime·starttheworld(void) } mp = mget(); if(mp == nil) { - pidleput(p); - break; + p->link = p1; + p1 = p; + continue; } if(mp->nextp) runtime·throw("starttheworld: inconsistent mp->nextp"); @@ -428,6 +433,13 @@ runtime·starttheworld(void) } runtime·unlock(&runtime·sched); + while(p1) { + p = p1; + p1 = p1->link; + add = false; + newm(nil, p); + } + if(add) { // If GC could have used another helper proc, start one now, // in the hope that it will be available next time. @@ -473,7 +485,7 @@ runtime·mstart(void) m->mstartfn(); if(m->helpgc) { - m->helpgc = false; + m->helpgc = 0; stopm(); } else if(m != &runtime·m0) { acquirep(m->nextp); @@ -782,8 +794,8 @@ retry: runtime·notesleep(&m->park); runtime·noteclear(&m->park); if(m->helpgc) { - m->helpgc = 0; runtime·gchelper(); + m->helpgc = 0; m->mcache = nil; goto retry; } @@ -970,7 +982,7 @@ execute(G *gp) } // Finds a runnable goroutine to execute. -// Tries to steal from other P's and get g from global queue. +// Tries to steal from other P's, get g from global queue, poll network. static G* findrunnable(void) { @@ -995,6 +1007,13 @@ top: if(gp) return gp; } + // poll network + gp = runtime·netpoll(false); // non-blocking + if(gp) { + injectglist(gp->schedlink); + gp->status = Grunnable; + return gp; + } // If number of spinning M's >= number of busy P's, block. // This is necessary to prevent excessive CPU consumption // when GOMAXPROCS>>1 but the program parallelism is low. @@ -1049,10 +1068,54 @@ stop: break; } } + // poll network + if(runtime·xchg64(&runtime·sched.lastpoll, 0) != 0) { + if(m->p) + runtime·throw("findrunnable: netpoll with p"); + if(m->spinning) + runtime·throw("findrunnable: netpoll with spinning"); + gp = runtime·netpoll(true); // block until new work is available + runtime·atomicstore64(&runtime·sched.lastpoll, runtime·nanotime()); + if(gp) { + runtime·lock(&runtime·sched); + p = pidleget(); + runtime·unlock(&runtime·sched); + if(p) { + acquirep(p); + injectglist(gp->schedlink); + gp->status = Grunnable; + return gp; + } + injectglist(gp); + } + } stopm(); goto top; } +// Injects the list of runnable G's into the scheduler. +// Can run concurrently with GC. +static void +injectglist(G *glist) +{ + int32 n; + G *gp; + + if(glist == nil) + return; + runtime·lock(&runtime·sched); + for(n = 0; glist; n++) { + gp = glist; + glist = gp->schedlink; + gp->status = Grunnable; + globrunqput(gp); + } + runtime·unlock(&runtime·sched); + + for(; n && runtime·sched.npidle; n--) + startm(nil, false); +} + // One round of scheduler: find a runnable goroutine and execute it. // Never returns. static void @@ -1164,6 +1227,11 @@ goexit0(G *gp) gp->lockedm = nil; m->curg = nil; m->lockedg = nil; + if(m->locked & ~LockExternal) { + runtime·printf("invalid m->locked = %d", m->locked); + runtime·throw("internal lockOSThread error"); + } + m->locked = 0; runtime·unwindstack(gp, nil); gfput(m->p, gp); schedule(); @@ -1251,7 +1319,7 @@ void p = releasep(); handoffp(p); - if(g == scvg) // do not consider blocked scavenger for deadlock detection + if(g->isbackground) // do not consider blocked scavenger for deadlock detection inclocked(1); runtime·gosave(&g->sched); // re-save for traceback } @@ -1283,7 +1351,7 @@ runtime·exitsyscall(void) return; } - if(g == scvg) // do not consider blocked scavenger for deadlock detection + if(g->isbackground) // do not consider blocked scavenger for deadlock detection inclocked(-1); // Try to get any other idle P. m->p = nil; @@ -1885,7 +1953,7 @@ checkdead(void) } grunning = 0; for(gp = runtime·allg; gp; gp = gp->alllink) { - if(gp == scvg) + if(gp->isbackground) continue; s = gp->status; if(s == Gwaiting) @@ -1905,6 +1973,8 @@ static void sysmon(void) { uint32 idle, delay; + int64 now, lastpoll; + G *gp; uint32 ticks[MaxGomaxprocs]; idle = 0; // how many cycles in succession we had not wokeup somebody @@ -1929,6 +1999,14 @@ sysmon(void) } else runtime·unlock(&runtime·sched); } + // poll network if not polled for more than 10ms + lastpoll = runtime·atomicload64(&runtime·sched.lastpoll); + now = runtime·nanotime(); + if(lastpoll != 0 && lastpoll + 10*1000*1000 > now) { + gp = runtime·netpoll(false); // non-blocking + injectglist(gp); + } + // retake P's blocked in syscalls if(retake(ticks)) idle = 0; else diff --git a/src/pkg/runtime/race/testdata/map_test.go b/src/pkg/runtime/race/testdata/map_test.go index 6f86a50b7..35db8db69 100644 --- a/src/pkg/runtime/race/testdata/map_test.go +++ b/src/pkg/runtime/race/testdata/map_test.go @@ -94,8 +94,7 @@ func TestNoRaceMapRangeRange(t *testing.T) { <-ch } -// Map len is not instrumented. -func TestRaceFailingMapLen(t *testing.T) { +func TestRaceMapLen(t *testing.T) { m := make(map[string]bool) ch := make(chan bool, 1) go func() { @@ -117,8 +116,7 @@ func TestRaceMapDelete(t *testing.T) { <-ch } -// Map len is not instrumented. -func TestRaceFailingMapLenDelete(t *testing.T) { +func TestRaceMapLenDelete(t *testing.T) { m := make(map[string]bool) ch := make(chan bool, 1) go func() { diff --git a/src/pkg/runtime/race/testdata/mop_test.go b/src/pkg/runtime/race/testdata/mop_test.go index f2daa3730..26cd3a4e4 100644 --- a/src/pkg/runtime/race/testdata/mop_test.go +++ b/src/pkg/runtime/race/testdata/mop_test.go @@ -306,6 +306,102 @@ func TestNoRacePlus(t *testing.T) { <-ch } +func TestRaceComplement(t *testing.T) { + var x, y, z int + ch := make(chan int, 2) + + go func() { + x = ^y + ch <- 1 + }() + go func() { + y = ^z + ch <- 1 + }() + <-ch + <-ch +} + +func TestRaceDiv(t *testing.T) { + var x, y, z int + ch := make(chan int, 2) + + go func() { + x = y / (z + 1) + ch <- 1 + }() + go func() { + y = z + ch <- 1 + }() + <-ch + <-ch +} + +func TestRaceDivConst(t *testing.T) { + var x, y, z int + ch := make(chan int, 2) + + go func() { + x = y / 3 + ch <- 1 + }() + go func() { + y = z + ch <- 1 + }() + <-ch + <-ch +} + +func TestRaceMod(t *testing.T) { + var x, y, z int + ch := make(chan int, 2) + + go func() { + x = y % (z + 1) + ch <- 1 + }() + go func() { + y = z + ch <- 1 + }() + <-ch + <-ch +} + +func TestRaceModConst(t *testing.T) { + var x, y, z int + ch := make(chan int, 2) + + go func() { + x = y % 3 + ch <- 1 + }() + go func() { + y = z + ch <- 1 + }() + <-ch + <-ch +} + +func TestRaceRotate(t *testing.T) { + var x, y, z uint32 + ch := make(chan int, 2) + + go func() { + x = y<<12 | y>>20 + ch <- 1 + }() + go func() { + y = z + ch <- 1 + }() + <-ch + <-ch +} + // May crash if the instrumentation is reckless. func TestNoRaceEnoughRegisters(t *testing.T) { // from erf.go diff --git a/src/pkg/runtime/race/testdata/regression_test.go b/src/pkg/runtime/race/testdata/regression_test.go index afe8cc5ec..f08ee3ed3 100644 --- a/src/pkg/runtime/race/testdata/regression_test.go +++ b/src/pkg/runtime/race/testdata/regression_test.go @@ -45,6 +45,18 @@ func InstrumentMapLen3() { _ = len(*m[0]) } +func TestRaceUnaddressableMapLen(t *testing.T) { + m := make(map[int]map[int]int) + ch := make(chan int, 1) + m[0] = make(map[int]int) + go func() { + _ = len(m[0]) + ch <- 0 + }() + m[0][0] = 1 + <-ch +} + type Rect struct { x, y int } diff --git a/src/pkg/runtime/race/testdata/slice_test.go b/src/pkg/runtime/race/testdata/slice_test.go index 773463662..1fe051b12 100644 --- a/src/pkg/runtime/race/testdata/slice_test.go +++ b/src/pkg/runtime/race/testdata/slice_test.go @@ -463,3 +463,24 @@ func TestRaceSliceRuneToString(t *testing.T) { s[9] = 42 <-c } + +func TestRaceConcatString(t *testing.T) { + s := "hello" + c := make(chan string, 1) + go func() { + c <- s + " world" + }() + s = "world" + <-c +} + +func TestRaceCompareString(t *testing.T) { + s1 := "hello" + s2 := "world" + c := make(chan bool, 1) + go func() { + c <- s1 == s2 + }() + s1 = s2 + <-c +} diff --git a/src/pkg/runtime/rt0_darwin_386.s b/src/pkg/runtime/rt0_darwin_386.s index 30b497f5e..4b4c1f294 100644 --- a/src/pkg/runtime/rt0_darwin_386.s +++ b/src/pkg/runtime/rt0_darwin_386.s @@ -2,7 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Darwin and Linux use the same linkage to main +TEXT _rt0_386_darwin(SB),7,$8 + MOVL 8(SP), AX + LEAL 12(SP), BX + MOVL AX, 0(SP) + MOVL BX, 4(SP) + CALL main(SB) + INT $3 -TEXT _rt0_386_darwin(SB),7,$0 +TEXT main(SB),7,$0 JMP _rt0_386(SB) diff --git a/src/pkg/runtime/rt0_darwin_amd64.s b/src/pkg/runtime/rt0_darwin_amd64.s index 4cfab5876..45e69a015 100644 --- a/src/pkg/runtime/rt0_darwin_amd64.s +++ b/src/pkg/runtime/rt0_darwin_amd64.s @@ -2,9 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Darwin and Linux use the same linkage to main - TEXT _rt0_amd64_darwin(SB),7,$-8 + LEAQ 8(SP), SI // argv + MOVQ 0(SP), DI // argc + MOVQ $main(SB), AX + JMP AX + +TEXT main(SB),7,$-8 MOVQ $_rt0_amd64(SB), AX - MOVQ SP, DI JMP AX diff --git a/src/pkg/runtime/rt0_freebsd_386.s b/src/pkg/runtime/rt0_freebsd_386.s index 3ca981b3a..c84482cdb 100644 --- a/src/pkg/runtime/rt0_freebsd_386.s +++ b/src/pkg/runtime/rt0_freebsd_386.s @@ -2,8 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Darwin and Linux use the same linkage to main +TEXT _rt0_386_freebsd(SB),7,$8 + MOVL 8(SP), AX + LEAL 12(SP), BX + MOVL AX, 0(SP) + MOVL BX, 4(SP) + CALL main(SB) + INT $3 -TEXT _rt0_386_freebsd(SB),7,$0 +TEXT main(SB),7,$0 JMP _rt0_386(SB) - diff --git a/src/pkg/runtime/rt0_freebsd_amd64.s b/src/pkg/runtime/rt0_freebsd_amd64.s index 5d2eeeeff..e6c6fb9ca 100644 --- a/src/pkg/runtime/rt0_freebsd_amd64.s +++ b/src/pkg/runtime/rt0_freebsd_amd64.s @@ -2,8 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Darwin and Linux use the same linkage to main - TEXT _rt0_amd64_freebsd(SB),7,$-8 - MOVQ $_rt0_amd64(SB), DX - JMP DX + LEAQ 8(DI), SI // argv + MOVQ 0(DI), DI // argc + MOVQ $main(SB), AX + JMP AX + +TEXT main(SB),7,$-8 + MOVQ $_rt0_amd64(SB), AX + JMP AX diff --git a/src/pkg/runtime/rt0_linux_386.s b/src/pkg/runtime/rt0_linux_386.s index 83149540e..73cca5d98 100644 --- a/src/pkg/runtime/rt0_linux_386.s +++ b/src/pkg/runtime/rt0_linux_386.s @@ -2,11 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Darwin and Linux use the same linkage to main - -TEXT _rt0_386_linux(SB),7,$0 +TEXT _rt0_386_linux(SB),7,$8 + MOVL 8(SP), AX + LEAL 12(SP), BX + MOVL AX, 0(SP) + MOVL BX, 4(SP) CALL runtime·linux_setup_vdso(SB) - JMP _rt0_386(SB) + CALL main(SB) + INT $3 + +TEXT main(SB),7,$0 + JMP _rt0_386(SB) TEXT _fallback_vdso(SB),7,$0 INT $0x80 diff --git a/src/pkg/runtime/rt0_linux_amd64.s b/src/pkg/runtime/rt0_linux_amd64.s index dac9ae181..dfc9c0421 100644 --- a/src/pkg/runtime/rt0_linux_amd64.s +++ b/src/pkg/runtime/rt0_linux_amd64.s @@ -2,9 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Darwin and Linux use the same linkage to main - TEXT _rt0_amd64_linux(SB),7,$-8 + LEAQ 8(SP), SI // argv + MOVQ 0(SP), DI // argc + MOVQ $main(SB), AX + JMP AX + +TEXT main(SB),7,$-8 MOVQ $_rt0_amd64(SB), AX - MOVQ SP, DI JMP AX diff --git a/src/pkg/runtime/rt0_netbsd_386.s b/src/pkg/runtime/rt0_netbsd_386.s index 829e4133b..b4c029c53 100644 --- a/src/pkg/runtime/rt0_netbsd_386.s +++ b/src/pkg/runtime/rt0_netbsd_386.s @@ -2,5 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -TEXT _rt0_386_netbsd(SB),7,$0 - JMP _rt0_386(SB) +TEXT _rt0_386_netbsd(SB),7,$8 + MOVL 8(SP), AX + LEAL 12(SP), BX + MOVL AX, 0(SP) + MOVL BX, 4(SP) + CALL main(SB) + INT $3 + +TEXT main(SB),7,$0 + JMP _rt0_386(SB) diff --git a/src/pkg/runtime/rt0_netbsd_amd64.s b/src/pkg/runtime/rt0_netbsd_amd64.s index 85482b98d..9e7b78edc 100644 --- a/src/pkg/runtime/rt0_netbsd_amd64.s +++ b/src/pkg/runtime/rt0_netbsd_amd64.s @@ -3,6 +3,11 @@ // license that can be found in the LICENSE file. TEXT _rt0_amd64_netbsd(SB),7,$-8 - MOVQ $_rt0_amd64(SB), DX - MOVQ SP, DI - JMP DX + LEAQ 8(SP), SI // argv + MOVQ 0(SP), DI // argc + MOVQ $main(SB), AX + JMP AX + +TEXT main(SB),7,$-8 + MOVQ $_rt0_amd64(SB), AX + JMP AX diff --git a/src/pkg/runtime/rt0_openbsd_386.s b/src/pkg/runtime/rt0_openbsd_386.s index e7e0da78f..9c00a7334 100644 --- a/src/pkg/runtime/rt0_openbsd_386.s +++ b/src/pkg/runtime/rt0_openbsd_386.s @@ -2,5 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -TEXT _rt0_386_openbsd(SB),7,$0 - JMP _rt0_386(SB) +TEXT _rt0_386_openbsd(SB),7,$8 + MOVL 8(SP), AX + LEAL 12(SP), BX + MOVL AX, 0(SP) + MOVL BX, 4(SP) + CALL main(SB) + INT $3 + +TEXT main(SB),7,$0 + JMP _rt0_386(SB) diff --git a/src/pkg/runtime/rt0_openbsd_amd64.s b/src/pkg/runtime/rt0_openbsd_amd64.s index e7fce5969..245a4c0f9 100644 --- a/src/pkg/runtime/rt0_openbsd_amd64.s +++ b/src/pkg/runtime/rt0_openbsd_amd64.s @@ -3,6 +3,11 @@ // license that can be found in the LICENSE file. TEXT _rt0_amd64_openbsd(SB),7,$-8 - MOVQ $_rt0_amd64(SB), DX - MOVQ SP, DI - JMP DX + LEAQ 8(SP), SI // argv + MOVQ 0(SP), DI // argc + MOVQ $main(SB), AX + JMP AX + +TEXT main(SB),7,$-8 + MOVQ $_rt0_amd64(SB), AX + JMP AX diff --git a/src/pkg/runtime/rt0_plan9_386.s b/src/pkg/runtime/rt0_plan9_386.s index 56f3a0f6c..7af1eae7c 100644 --- a/src/pkg/runtime/rt0_plan9_386.s +++ b/src/pkg/runtime/rt0_plan9_386.s @@ -26,6 +26,13 @@ argv_fix: LOOP argv_fix CALL runtime·asminit(SB) + + MOVL 0(SP), AX + LEAL 4(SP), BX + PUSHL BX + PUSHL AX + PUSHL $-1 + JMP _rt0_386(SB) DATA runtime·isplan9(SB)/4, $1 diff --git a/src/pkg/runtime/rt0_plan9_amd64.s b/src/pkg/runtime/rt0_plan9_amd64.s index 2b1fa2ae1..16e5e82b7 100644 --- a/src/pkg/runtime/rt0_plan9_amd64.s +++ b/src/pkg/runtime/rt0_plan9_amd64.s @@ -2,9 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -TEXT _rt0_amd64_plan9(SB),7, $0 +TEXT _rt0_amd64_plan9(SB),7,$-8 + LEAQ 8(SP), SI // argv + MOVQ 0(SP), DI // argc MOVQ $_rt0_amd64(SB), AX - MOVQ SP, DI JMP AX DATA runtime·isplan9(SB)/4, $1 diff --git a/src/pkg/runtime/rt0_windows_386.s b/src/pkg/runtime/rt0_windows_386.s index a06aa787e..6e34c6c17 100644 --- a/src/pkg/runtime/rt0_windows_386.s +++ b/src/pkg/runtime/rt0_windows_386.s @@ -2,8 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -TEXT _rt0_386_windows(SB),7,$0 +TEXT _rt0_386_windows(SB),7,$12 + MOVL 12(SP), AX + LEAL 16(SP), BX + MOVL AX, 4(SP) + MOVL BX, 8(SP) + MOVL $-1, 0(SP) // return PC for main + JMP main(SB) + +TEXT main(SB),7,$0 JMP _rt0_386(SB) + DATA runtime·iswindows(SB)/4, $1 GLOBL runtime·iswindows(SB), $4 diff --git a/src/pkg/runtime/rt0_windows_amd64.s b/src/pkg/runtime/rt0_windows_amd64.s index dc1408adc..b48c05570 100644 --- a/src/pkg/runtime/rt0_windows_amd64.s +++ b/src/pkg/runtime/rt0_windows_amd64.s @@ -4,9 +4,14 @@ #include "zasm_GOOS_GOARCH.h" -TEXT _rt0_amd64_windows(SB),7,$-8 +TEXT _rt0_amd64_windows(SB),7,$-8 + LEAQ 8(SP), SI // argv + MOVQ 0(SP), DI // argc + MOVQ $main(SB), AX + JMP AX + +TEXT main(SB),7,$-8 MOVQ $_rt0_amd64(SB), AX - MOVQ SP, DI JMP AX DATA runtime·iswindows(SB)/4, $1 diff --git a/src/pkg/runtime/runtime.c b/src/pkg/runtime/runtime.c index 4d57cbafd..d62408118 100644 --- a/src/pkg/runtime/runtime.c +++ b/src/pkg/runtime/runtime.c @@ -17,21 +17,34 @@ enum { */ void runtime·sigpanic(void); +// The GOTRACEBACK environment variable controls the +// behavior of a Go program that is crashing and exiting. +// GOTRACEBACK=0 suppress all tracebacks +// GOTRACEBACK=1 default behavior - show tracebacks but exclude runtime frames +// GOTRACEBACK=2 show tracebacks including runtime frames +// GOTRACEBACK=crash show tracebacks including runtime frames, then crash (core dump etc) int32 -runtime·gotraceback(void) +runtime·gotraceback(bool *crash) { byte *p; + if(crash != nil) + *crash = false; p = runtime·getenv("GOTRACEBACK"); if(p == nil || p[0] == '\0') return 1; // default is on + if(runtime·strcmp(p, (byte*)"crash") == 0) { + if(crash != nil) + *crash = true; + return 2; // extra information + } return runtime·atoi(p); } int32 -runtime·mcmp(byte *s1, byte *s2, uint32 n) +runtime·mcmp(byte *s1, byte *s2, uintptr n) { - uint32 i; + uintptr i; byte c1, c2; for(i=0; i<n; i++) { @@ -75,6 +88,11 @@ runtime·args(int32 c, uint8 **v) int32 runtime·isplan9; int32 runtime·iswindows; +// Information about what cpu features are available. +// Set on startup in asm_{x86/amd64}.s. +uint32 runtime·cpuid_ecx; +uint32 runtime·cpuid_edx; + void runtime·goargs(void) { @@ -156,6 +174,10 @@ TestAtomic64(void) runtime·throw("xadd64 failed"); if(runtime·atomicload64(&z64) != (2ull<<40)+2) runtime·throw("xadd64 failed"); + if(runtime·xchg64(&z64, (3ull<<40)+3) != (2ull<<40)+2) + runtime·throw("xchg64 failed"); + if(runtime·atomicload64(&z64) != (3ull<<40)+3) + runtime·throw("xchg64 failed"); } void diff --git a/src/pkg/runtime/runtime.h b/src/pkg/runtime/runtime.h index 08f43a69b..46c77e3fd 100644 --- a/src/pkg/runtime/runtime.h +++ b/src/pkg/runtime/runtime.h @@ -85,6 +85,7 @@ typedef struct LFNode LFNode; typedef struct ParFor ParFor; typedef struct ParForThread ParForThread; typedef struct CgoMal CgoMal; +typedef struct PollDesc PollDesc; /* * Per-CPU declaration. @@ -235,8 +236,9 @@ struct G int8* waitreason; // if status==Gwaiting G* schedlink; bool ispanic; - bool issystem; - int8 raceignore; // ignore race detection events + bool issystem; // do not output in stack dump + bool isbackground; // ignore in deadlock detector + int8 raceignore; // ignore race detection events M* m; // for debuggers, but offset not hard-coded M* lockedm; int32 sig; @@ -320,6 +322,7 @@ struct M #endif #ifdef GOOS_plan9 int8* notesig; + byte* errstr; #endif SEH* seh; uintptr end[]; @@ -381,6 +384,8 @@ enum SigThrow = 1<<2, // if signal.Notify doesn't take it, exit loudly SigPanic = 1<<3, // if the signal is from the kernel, panic SigDefault = 1<<4, // if the signal isn't explicitly requested, don't monitor it + SigHandling = 1<<5, // our signal handler is registered + SigIgnored = 1<<6, // the signal was ignored before we registered for it }; // NOTE(rsc): keep in sync with extern.go:/type.Func. @@ -482,6 +487,7 @@ struct ParFor bool wait; // if true, wait while all threads finish processing, // otherwise parfor may return while other threads are still working ParForThread *thr; // array of thread descriptors + uint32 pad; // to align ParForThread.pos for 64-bit atomic operations // stats uint64 nsteal; uint64 nstealcnt; @@ -555,11 +561,25 @@ struct Alg extern Alg runtime·algarray[Amax]; +byte* runtime·startup_random_data; +uint32 runtime·startup_random_data_len; +void runtime·get_random_data(byte**, int32*); + +enum { + // hashinit wants this many random bytes + HashRandomBytes = 32 +}; +void runtime·hashinit(void); + void runtime·memhash(uintptr*, uintptr, void*); void runtime·nohash(uintptr*, uintptr, void*); void runtime·strhash(uintptr*, uintptr, void*); void runtime·interhash(uintptr*, uintptr, void*); void runtime·nilinterhash(uintptr*, uintptr, void*); +void runtime·aeshash(uintptr*, uintptr, void*); +void runtime·aeshash32(uintptr*, uintptr, void*); +void runtime·aeshash64(uintptr*, uintptr, void*); +void runtime·aeshashstr(uintptr*, uintptr, void*); void runtime·memequal(bool*, uintptr, void*, void*); void runtime·noequal(bool*, uintptr, void*, void*); @@ -578,7 +598,6 @@ void runtime·memcopy16(uintptr, void*, void*); void runtime·memcopy32(uintptr, void*, void*); void runtime·memcopy64(uintptr, void*, void*); void runtime·memcopy128(uintptr, void*, void*); -void runtime·memcopy(uintptr, void*, void*); void runtime·strcopy(uintptr, void*, void*); void runtime·algslicecopy(uintptr, void*, void*); void runtime·intercopy(uintptr, void*, void*); @@ -635,6 +654,8 @@ extern bool runtime·iscgo; extern void (*runtime·sysargs)(int32, uint8**); extern uint32 runtime·maxstring; extern uint32 runtime·Hchansize; +extern uint32 runtime·cpuid_ecx; +extern uint32 runtime·cpuid_edx; /* * common functions and data @@ -666,8 +687,8 @@ void runtime·panicstring(int8*); void runtime·prints(int8*); void runtime·printf(int8*, ...); byte* runtime·mchr(byte*, byte, byte*); -int32 runtime·mcmp(byte*, byte*, uint32); -void runtime·memmove(void*, void*, uint32); +int32 runtime·mcmp(byte*, byte*, uintptr); +void runtime·memmove(void*, void*, uintptr); void* runtime·mal(uintptr); String runtime·catstring(String, String); String runtime·gostring(byte*); @@ -677,11 +698,15 @@ String runtime·gostringnocopy(byte*); String runtime·gostringw(uint16*); void runtime·initsig(void); void runtime·sigenable(uint32 sig); -int32 runtime·gotraceback(void); +void runtime·sigdisable(uint32 sig); +int32 runtime·gotraceback(bool *crash); void runtime·goroutineheader(G*); void runtime·traceback(uint8 *pc, uint8 *sp, uint8 *lr, G* gp); void runtime·tracebackothers(G*); +int32 runtime·open(int8*, int32, int32); +int32 runtime·read(int32, void*, int32); int32 runtime·write(int32, void*, int32); +int32 runtime·close(int32); int32 runtime·mincore(void*, uintptr, byte*); bool runtime·cas(uint32*, uint32, uint32); bool runtime·cas64(uint64*, uint64*, uint64); @@ -691,6 +716,7 @@ bool runtime·casp(void**, void*, void*); uint32 runtime·xadd(uint32 volatile*, int32); uint64 runtime·xadd64(uint64 volatile*, int64); uint32 runtime·xchg(uint32 volatile*, uint32); +uint64 runtime·xchg64(uint64 volatile*, uint64); uint32 runtime·atomicload(uint32 volatile*); void runtime·atomicstore(uint32 volatile*, uint32); void runtime·atomicstore64(uint64 volatile*, uint64); @@ -761,6 +787,14 @@ int64 runtime·cputicks(void); int64 runtime·tickspersecond(void); void runtime·blockevent(int64, int32); extern int64 runtime·blockprofilerate; +void runtime·addtimer(Timer*); +bool runtime·deltimer(Timer*); +G* runtime·netpoll(bool); +void runtime·netpollinit(void); +int32 runtime·netpollopen(int32, PollDesc*); +int32 runtime·netpollclose(int32); +void runtime·netpollready(G**, PollDesc*, int32); +void runtime·crash(void); #pragma varargck argpos runtime·printf 1 #pragma varargck type "d" int32 @@ -929,7 +963,6 @@ void runtime·mapassign(MapType*, Hmap*, byte*, byte*); void runtime·mapaccess(MapType*, Hmap*, byte*, byte*, bool*); void runtime·mapiternext(struct hash_iter*); bool runtime·mapiterkey(struct hash_iter*, void*); -void runtime·mapiterkeyvalue(struct hash_iter*, void*, void*); Hmap* runtime·makemap_c(MapType*, int64); Hchan* runtime·makechan_c(ChanType*, int64); diff --git a/src/pkg/runtime/signal_386.c b/src/pkg/runtime/signal_386.c new file mode 100644 index 000000000..72b4a66f8 --- /dev/null +++ b/src/pkg/runtime/signal_386.c @@ -0,0 +1,123 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux netbsd openbsd + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" +#include "signal_GOOS_GOARCH.h" +#include "signals_GOOS.h" + +void +runtime·dumpregs(Siginfo *info, void *ctxt) +{ + USED(info); + USED(ctxt); + + runtime·printf("eax %x\n", SIG_EAX(info, ctxt)); + runtime·printf("ebx %x\n", SIG_EBX(info, ctxt)); + runtime·printf("ecx %x\n", SIG_ECX(info, ctxt)); + runtime·printf("edx %x\n", SIG_EDX(info, ctxt)); + runtime·printf("edi %x\n", SIG_EDI(info, ctxt)); + runtime·printf("esi %x\n", SIG_ESI(info, ctxt)); + runtime·printf("ebp %x\n", SIG_EBP(info, ctxt)); + runtime·printf("esp %x\n", SIG_ESP(info, ctxt)); + runtime·printf("eip %x\n", SIG_EIP(info, ctxt)); + runtime·printf("eflags %x\n", SIG_EFLAGS(info, ctxt)); + runtime·printf("cs %x\n", SIG_CS(info, ctxt)); + runtime·printf("fs %x\n", SIG_FS(info, ctxt)); + runtime·printf("gs %x\n", SIG_GS(info, ctxt)); +} + +void +runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) +{ + uintptr *sp; + SigTab *t; + bool crash; + + if(sig == SIGPROF) { + if(gp != m->g0 && gp != m->gsignal) + runtime·sigprof((byte*)SIG_EIP(info, ctxt), (byte*)SIG_ESP(info, ctxt), nil, gp); + return; + } + + t = &runtime·sigtab[sig]; + if(SIG_CODE0(info, ctxt) != SI_USER && (t->flags & SigPanic)) { + if(gp == nil || gp == m->g0) + goto Throw; + + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp->sig = sig; + gp->sigcode0 = SIG_CODE0(info, ctxt); + gp->sigcode1 = SIG_CODE1(info, ctxt); + gp->sigpc = SIG_EIP(info, ctxt); + +#ifdef GOOS_darwin + // Work around Leopard bug that doesn't set FPE_INTDIV. + // Look at instruction to see if it is a divide. + // Not necessary in Snow Leopard (si_code will be != 0). + if(sig == SIGFPE && gp->sigcode0 == 0) { + byte *pc; + pc = (byte*)gp->sigpc; + if(pc[0] == 0x66) // 16-bit instruction prefix + pc++; + if(pc[0] == 0xF6 || pc[0] == 0xF7) + gp->sigcode0 = FPE_INTDIV; + } +#endif + + // Only push runtime·sigpanic if eip != 0. + // If eip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime·sigpanic instead. + // (Otherwise the trace will end at runtime·sigpanic and we + // won't get to see who faulted.) + if(SIG_EIP(info, ctxt) != 0) { + sp = (uintptr*)SIG_ESP(info, ctxt); + *--sp = SIG_EIP(info, ctxt); + SIG_ESP(info, ctxt) = (uintptr)sp; + } + SIG_EIP(info, ctxt) = (uintptr)runtime·sigpanic; + return; + } + + if(SIG_CODE0(info, ctxt) == SI_USER || (t->flags & SigNotify)) + if(runtime·sigsend(sig)) + return; + if(t->flags & SigKill) + runtime·exit(2); + if(!(t->flags & SigThrow)) + return; + +Throw: + runtime·startpanic(); + + if(sig < 0 || sig >= NSIG) + runtime·printf("Signal %d\n", sig); + else + runtime·printf("%s\n", runtime·sigtab[sig].name); + + runtime·printf("PC=%x\n", SIG_EIP(info, ctxt)); + if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { + runtime·printf("signal arrived during cgo execution\n"); + gp = m->lockedg; + } + runtime·printf("\n"); + + if(runtime·gotraceback(&crash)){ + runtime·traceback((void*)SIG_EIP(info, ctxt), (void*)SIG_ESP(info, ctxt), 0, gp); + runtime·tracebackothers(gp); + runtime·dumpregs(info, ctxt); + } + + if(crash) + runtime·crash(); + + runtime·exit(2); +} diff --git a/src/pkg/runtime/signal_amd64.c b/src/pkg/runtime/signal_amd64.c new file mode 100644 index 000000000..ce17bf36d --- /dev/null +++ b/src/pkg/runtime/signal_amd64.c @@ -0,0 +1,133 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux netbsd openbsd + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" +#include "signal_GOOS_GOARCH.h" +#include "signals_GOOS.h" + +void +runtime·dumpregs(Siginfo *info, void *ctxt) +{ + USED(info); + USED(ctxt); + + runtime·printf("rax %X\n", SIG_RAX(info, ctxt)); + runtime·printf("rbx %X\n", SIG_RBX(info, ctxt)); + runtime·printf("rcx %X\n", SIG_RCX(info, ctxt)); + runtime·printf("rdx %X\n", SIG_RDX(info, ctxt)); + runtime·printf("rdi %X\n", SIG_RDI(info, ctxt)); + runtime·printf("rsi %X\n", SIG_RSI(info, ctxt)); + runtime·printf("rbp %X\n", SIG_RBP(info, ctxt)); + runtime·printf("rsp %X\n", SIG_RSP(info, ctxt)); + runtime·printf("r8 %X\n", SIG_R8(info, ctxt) ); + runtime·printf("r9 %X\n", SIG_R9(info, ctxt) ); + runtime·printf("r10 %X\n", SIG_R10(info, ctxt)); + runtime·printf("r11 %X\n", SIG_R11(info, ctxt)); + runtime·printf("r12 %X\n", SIG_R12(info, ctxt)); + runtime·printf("r13 %X\n", SIG_R13(info, ctxt)); + runtime·printf("r14 %X\n", SIG_R14(info, ctxt)); + runtime·printf("r15 %X\n", SIG_R15(info, ctxt)); + runtime·printf("rip %X\n", SIG_RIP(info, ctxt)); + runtime·printf("rflags %X\n", SIG_RFLAGS(info, ctxt)); + runtime·printf("cs %X\n", SIG_CS(info, ctxt)); + runtime·printf("fs %X\n", SIG_FS(info, ctxt)); + runtime·printf("gs %X\n", SIG_GS(info, ctxt)); +} + +void +runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) +{ + uintptr *sp; + SigTab *t; + bool crash; + + if(sig == SIGPROF) { + if(gp != m->g0 && gp != m->gsignal) + runtime·sigprof((byte*)SIG_RIP(info, ctxt), (byte*)SIG_RSP(info, ctxt), nil, gp); + return; + } + + t = &runtime·sigtab[sig]; + if(SIG_CODE0(info, ctxt) != SI_USER && (t->flags & SigPanic)) { + if(gp == nil || gp == m->g0) + goto Throw; + + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp->sig = sig; + gp->sigcode0 = SIG_CODE0(info, ctxt); + gp->sigcode1 = SIG_CODE1(info, ctxt); + gp->sigpc = SIG_RIP(info, ctxt); + +#ifdef GOOS_darwin + // Work around Leopard bug that doesn't set FPE_INTDIV. + // Look at instruction to see if it is a divide. + // Not necessary in Snow Leopard (si_code will be != 0). + if(sig == SIGFPE && gp->sigcode0 == 0) { + byte *pc; + pc = (byte*)gp->sigpc; + if((pc[0]&0xF0) == 0x40) // 64-bit REX prefix + pc++; + else if(pc[0] == 0x66) // 16-bit instruction prefix + pc++; + if(pc[0] == 0xF6 || pc[0] == 0xF7) + gp->sigcode0 = FPE_INTDIV; + } +#endif + + // Only push runtime·sigpanic if rip != 0. + // If rip == 0, probably panicked because of a + // call to a nil func. Not pushing that onto sp will + // make the trace look like a call to runtime·sigpanic instead. + // (Otherwise the trace will end at runtime·sigpanic and we + // won't get to see who faulted.) + if(SIG_RIP(info, ctxt) != 0) { + sp = (uintptr*)SIG_RSP(info, ctxt); + *--sp = SIG_RIP(info, ctxt); + SIG_RSP(info, ctxt) = (uintptr)sp; + } + SIG_RIP(info, ctxt) = (uintptr)runtime·sigpanic; + return; + } + + if(SIG_CODE0(info, ctxt) == SI_USER || (t->flags & SigNotify)) + if(runtime·sigsend(sig)) + return; + if(t->flags & SigKill) + runtime·exit(2); + if(!(t->flags & SigThrow)) + return; + +Throw: + runtime·startpanic(); + + if(sig < 0 || sig >= NSIG) + runtime·printf("Signal %d\n", sig); + else + runtime·printf("%s\n", runtime·sigtab[sig].name); + + runtime·printf("PC=%X\n", SIG_RIP(info, ctxt)); + if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { + runtime·printf("signal arrived during cgo execution\n"); + gp = m->lockedg; + } + runtime·printf("\n"); + + if(runtime·gotraceback(&crash)){ + runtime·traceback((void*)SIG_RIP(info, ctxt), (void*)SIG_RSP(info, ctxt), 0, gp); + runtime·tracebackothers(gp); + runtime·dumpregs(info, ctxt); + } + + if(crash) + runtime·crash(); + + runtime·exit(2); +} diff --git a/src/pkg/runtime/signal_arm.c b/src/pkg/runtime/signal_arm.c new file mode 100644 index 000000000..adf61de6b --- /dev/null +++ b/src/pkg/runtime/signal_arm.c @@ -0,0 +1,124 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux netbsd openbsd + +#include "runtime.h" +#include "defs_GOOS_GOARCH.h" +#include "os_GOOS.h" +#include "signal_GOOS_GOARCH.h" +#include "signals_GOOS.h" + +void +runtime·dumpregs(Siginfo *info, void *ctxt) +{ + USED(info); + USED(ctxt); + + runtime·printf("trap %x\n", SIG_TRAP(info, ctxt)); + runtime·printf("error %x\n", SIG_ERROR(info, ctxt)); + runtime·printf("oldmask %x\n", SIG_OLDMASK(info, ctxt)); + runtime·printf("r0 %x\n", SIG_R0(info, ctxt)); + runtime·printf("r1 %x\n", SIG_R1(info, ctxt)); + runtime·printf("r2 %x\n", SIG_R2(info, ctxt)); + runtime·printf("r3 %x\n", SIG_R3(info, ctxt)); + runtime·printf("r4 %x\n", SIG_R4(info, ctxt)); + runtime·printf("r5 %x\n", SIG_R5(info, ctxt)); + runtime·printf("r6 %x\n", SIG_R6(info, ctxt)); + runtime·printf("r7 %x\n", SIG_R7(info, ctxt)); + runtime·printf("r8 %x\n", SIG_R8(info, ctxt)); + runtime·printf("r9 %x\n", SIG_R9(info, ctxt)); + runtime·printf("r10 %x\n", SIG_R10(info, ctxt)); + runtime·printf("fp %x\n", SIG_FP(info, ctxt)); + runtime·printf("ip %x\n", SIG_IP(info, ctxt)); + runtime·printf("sp %x\n", SIG_SP(info, ctxt)); + runtime·printf("lr %x\n", SIG_LR(info, ctxt)); + runtime·printf("pc %x\n", SIG_PC(info, ctxt)); + runtime·printf("cpsr %x\n", SIG_CPSR(info, ctxt)); + runtime·printf("fault %x\n", SIG_FAULT(info, ctxt)); +} + +void +runtime·sighandler(int32 sig, Siginfo *info, void *ctxt, G *gp) +{ + SigTab *t; + bool crash; + + if(sig == SIGPROF) { + if(gp != m->g0 && gp != m->gsignal) + runtime·sigprof((uint8*)SIG_PC(info, ctxt), (uint8*)SIG_SP(info, ctxt), (uint8*)SIG_LR(info, ctxt), gp); + return; + } + + t = &runtime·sigtab[sig]; + if(SIG_CODE0(info, ctxt) != SI_USER && (t->flags & SigPanic)) { + if(gp == nil || gp == m->g0) + goto Throw; + + // Make it look like a call to the signal func. + // Have to pass arguments out of band since + // augmenting the stack frame would break + // the unwinding code. + gp->sig = sig; + gp->sigcode0 = SIG_CODE0(info, ctxt); + gp->sigcode1 = SIG_FAULT(info, ctxt); + gp->sigpc = SIG_PC(info, ctxt); + + // We arrange lr, and pc to pretend the panicking + // function calls sigpanic directly. + // Always save LR to stack so that panics in leaf + // functions are correctly handled. This smashes + // the stack frame but we're not going back there + // anyway. + SIG_SP(info, ctxt) -= 4; + *(uint32*)SIG_SP(info, ctxt) = SIG_LR(info, ctxt); + // Don't bother saving PC if it's zero, which is + // probably a call to a nil func: the old link register + // is more useful in the stack trace. + if(gp->sigpc != 0) + SIG_LR(info, ctxt) = gp->sigpc; + // In case we are panicking from external C code + SIG_R10(info, ctxt) = (uintptr)gp; + SIG_R9(info, ctxt) = (uintptr)m; + SIG_PC(info, ctxt) = (uintptr)runtime·sigpanic; + return; + } + + if(SIG_CODE0(info, ctxt) == SI_USER || (t->flags & SigNotify)) + if(runtime·sigsend(sig)) + return; + if(t->flags & SigKill) + runtime·exit(2); + if(!(t->flags & SigThrow)) + return; + +Throw: + if(runtime·panicking) // traceback already printed + runtime·exit(2); + runtime·panicking = 1; + + if(sig < 0 || sig >= NSIG) + runtime·printf("Signal %d\n", sig); + else + runtime·printf("%s\n", runtime·sigtab[sig].name); + + runtime·printf("PC=%x\n", SIG_PC(info, ctxt)); + if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { + runtime·printf("signal arrived during cgo execution\n"); + gp = m->lockedg; + } + runtime·printf("\n"); + + if(runtime·gotraceback(&crash)){ + runtime·traceback((void*)SIG_PC(info, ctxt), (void*)SIG_SP(info, ctxt), (void*)SIG_LR(info, ctxt), gp); + runtime·tracebackothers(gp); + runtime·printf("\n"); + runtime·dumpregs(info, ctxt); + } + + if(crash) + runtime·crash(); + + runtime·exit(2); +} diff --git a/src/pkg/runtime/signal_darwin_386.c b/src/pkg/runtime/signal_darwin_386.c deleted file mode 100644 index 132ca931b..000000000 --- a/src/pkg/runtime/signal_darwin_386.c +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signals_GOOS.h" - -void -runtime·dumpregs(Regs32 *r) -{ - runtime·printf("eax %x\n", r->eax); - runtime·printf("ebx %x\n", r->ebx); - runtime·printf("ecx %x\n", r->ecx); - runtime·printf("edx %x\n", r->edx); - runtime·printf("edi %x\n", r->edi); - runtime·printf("esi %x\n", r->esi); - runtime·printf("ebp %x\n", r->ebp); - runtime·printf("esp %x\n", r->esp); - runtime·printf("eip %x\n", r->eip); - runtime·printf("eflags %x\n", r->eflags); - runtime·printf("cs %x\n", r->cs); - runtime·printf("fs %x\n", r->fs); - runtime·printf("gs %x\n", r->gs); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Mcontext32 *mc; - Regs32 *r; - uintptr *sp; - byte *pc; - SigTab *t; - - uc = context; - mc = uc->uc_mcontext; - r = &mc->ss; - - if(sig == SIGPROF) { - if(gp != m->g0 && gp != m->gsignal) - runtime·sigprof((uint8*)r->eip, (uint8*)r->esp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Work around Leopard bug that doesn't set FPE_INTDIV. - // Look at instruction to see if it is a divide. - // Not necessary in Snow Leopard (si_code will be != 0). - if(sig == SIGFPE && info->si_code == 0) { - pc = (byte*)r->eip; - if(pc[0] == 0x66) // 16-bit instruction prefix - pc++; - if(pc[0] == 0xF6 || pc[0] == 0xF7) - info->si_code = FPE_INTDIV; - } - - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = (uintptr)info->si_addr; - gp->sigpc = r->eip; - - // Only push runtime·sigpanic if r->eip != 0. - // If r->eip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->eip != 0) { - sp = (uintptr*)r->esp; - *--sp = r->eip; - r->esp = (uintptr)sp; - } - r->eip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG){ - runtime·printf("Signal %d\n", sig); - }else{ - runtime·printf("%s\n", runtime·sigtab[sig].name); - } - - runtime·printf("PC=%x\n", r->eip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->eip, (void*)r->esp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(*(void**)sa.__sigaction_u == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0U; - sa.sa_tramp = (void*)runtime·sigtramp; // runtime·sigtramp's job is to call into real handler - *(uintptr*)sa.__sigaction_u = (uintptr)fn; - runtime·sigaction(i, &sa, nil); -} diff --git a/src/pkg/runtime/signal_darwin_386.h b/src/pkg/runtime/signal_darwin_386.h new file mode 100644 index 000000000..5459e10a1 --- /dev/null +++ b/src/pkg/runtime/signal_darwin_386.h @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext->ss) + +#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).eax) +#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).ebx) +#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).ecx) +#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).edx) +#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).edi) +#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).esi) +#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).ebp) +#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).esp) +#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).eip) +#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).eflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/pkg/runtime/signal_darwin_amd64.c b/src/pkg/runtime/signal_darwin_amd64.c deleted file mode 100644 index 4b7256bf4..000000000 --- a/src/pkg/runtime/signal_darwin_amd64.c +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "os_GOOS.h" -#include "signals_GOOS.h" - -void -runtime·dumpregs(Regs64 *r) -{ - runtime·printf("rax %X\n", r->rax); - runtime·printf("rbx %X\n", r->rbx); - runtime·printf("rcx %X\n", r->rcx); - runtime·printf("rdx %X\n", r->rdx); - runtime·printf("rdi %X\n", r->rdi); - runtime·printf("rsi %X\n", r->rsi); - runtime·printf("rbp %X\n", r->rbp); - runtime·printf("rsp %X\n", r->rsp); - runtime·printf("r8 %X\n", r->r8 ); - runtime·printf("r9 %X\n", r->r9 ); - runtime·printf("r10 %X\n", r->r10); - runtime·printf("r11 %X\n", r->r11); - runtime·printf("r12 %X\n", r->r12); - runtime·printf("r13 %X\n", r->r13); - runtime·printf("r14 %X\n", r->r14); - runtime·printf("r15 %X\n", r->r15); - runtime·printf("rip %X\n", r->rip); - runtime·printf("rflags %X\n", r->rflags); - runtime·printf("cs %X\n", r->cs); - runtime·printf("fs %X\n", r->fs); - runtime·printf("gs %X\n", r->gs); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Mcontext64 *mc; - Regs64 *r; - uintptr *sp; - byte *pc; - SigTab *t; - - uc = context; - mc = uc->uc_mcontext; - r = &mc->ss; - - if(sig == SIGPROF) { - if(gp != m->g0 && gp != m->gsignal) - runtime·sigprof((uint8*)r->rip, (uint8*)r->rsp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Work around Leopard bug that doesn't set FPE_INTDIV. - // Look at instruction to see if it is a divide. - // Not necessary in Snow Leopard (si_code will be != 0). - if(sig == SIGFPE && info->si_code == 0) { - pc = (byte*)r->rip; - if((pc[0]&0xF0) == 0x40) // 64-bit REX prefix - pc++; - else if(pc[0] == 0x66) // 16-bit instruction prefix - pc++; - if(pc[0] == 0xF6 || pc[0] == 0xF7) - info->si_code = FPE_INTDIV; - } - - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = (uintptr)info->si_addr; - gp->sigpc = r->rip; - - // Only push runtime·sigpanic if r->rip != 0. - // If r->rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->rip != 0) { - sp = (uintptr*)r->rsp; - *--sp = r->rip; - r->rsp = (uintptr)sp; - } - r->rip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG){ - runtime·printf("Signal %d\n", sig); - }else{ - runtime·printf("%s\n", runtime·sigtab[sig].name); - } - - runtime·printf("PC=%X\n", r->rip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->rip, (void*)r->rsp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - StackT st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(*(void**)sa.__sigaction_u == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - sa.sa_tramp = runtime·sigtramp; // runtime·sigtramp's job is to call into real handler - *(uintptr*)sa.__sigaction_u = (uintptr)fn; - runtime·sigaction(i, &sa, nil); -} diff --git a/src/pkg/runtime/signal_darwin_amd64.h b/src/pkg/runtime/signal_darwin_amd64.h new file mode 100644 index 000000000..e3da6de3a --- /dev/null +++ b/src/pkg/runtime/signal_darwin_amd64.h @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext->ss) + +#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).rax) +#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).rbx) +#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).rcx) +#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).rdx) +#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).rdi) +#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).rsi) +#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).rbp) +#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).rsp) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).r8) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).r9) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).r10) +#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).r11) +#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).r12) +#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).r13) +#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).r14) +#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).r15) +#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).rip) +#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).rflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/pkg/runtime/signal_freebsd_386.c b/src/pkg/runtime/signal_freebsd_386.c deleted file mode 100644 index 254e5e277..000000000 --- a/src/pkg/runtime/signal_freebsd_386.c +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - int32 sa_flags; /* see signal options below */ - Sigset sa_mask; /* signal mask to apply */ -} Sigaction; - -void -runtime·dumpregs(Mcontext *r) -{ - runtime·printf("eax %x\n", r->mc_eax); - runtime·printf("ebx %x\n", r->mc_ebx); - runtime·printf("ecx %x\n", r->mc_ecx); - runtime·printf("edx %x\n", r->mc_edx); - runtime·printf("edi %x\n", r->mc_edi); - runtime·printf("esi %x\n", r->mc_esi); - runtime·printf("ebp %x\n", r->mc_ebp); - runtime·printf("esp %x\n", r->mc_esp); - runtime·printf("eip %x\n", r->mc_eip); - runtime·printf("eflags %x\n", r->mc_eflags); - runtime·printf("cs %x\n", r->mc_cs); - runtime·printf("fs %x\n", r->mc_fs); - runtime·printf("gs %x\n", r->mc_gs); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Mcontext *r; - uintptr *sp; - SigTab *t; - - uc = context; - r = &uc->uc_mcontext; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->mc_eip, (uint8*)r->mc_esp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = (uintptr)info->si_addr; - gp->sigpc = r->mc_eip; - - // Only push runtime·sigpanic if r->mc_eip != 0. - // If r->mc_eip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->mc_eip != 0) { - sp = (uintptr*)r->mc_esp; - *--sp = r->mc_eip; - r->mc_esp = (uintptr)sp; - } - r->mc_eip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", r->mc_eip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->mc_eip, (void*)r->mc_esp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = (int8*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa.__sigaction_u.__sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask.__bits[0] = ~(uint32)0; - sa.sa_mask.__bits[1] = ~(uint32)0; - sa.sa_mask.__bits[2] = ~(uint32)0; - sa.sa_mask.__bits[3] = ~(uint32)0; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} diff --git a/src/pkg/runtime/signal_freebsd_386.h b/src/pkg/runtime/signal_freebsd_386.h new file mode 100644 index 000000000..a24f1ee96 --- /dev/null +++ b/src/pkg/runtime/signal_freebsd_386.h @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) + +#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).mc_eax) +#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).mc_ebx) +#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).mc_ecx) +#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).mc_edx) +#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).mc_edi) +#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).mc_esi) +#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).mc_ebp) +#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).mc_esp) +#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).mc_eip) +#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).mc_eflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).mc_cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).mc_fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).mc_gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/pkg/runtime/signal_freebsd_amd64.c b/src/pkg/runtime/signal_freebsd_amd64.c deleted file mode 100644 index 7dbf36075..000000000 --- a/src/pkg/runtime/signal_freebsd_amd64.c +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - int32 sa_flags; /* see signal options below */ - Sigset sa_mask; /* signal mask to apply */ -} Sigaction; - -void -runtime·dumpregs(Mcontext *r) -{ - runtime·printf("rax %X\n", r->mc_rax); - runtime·printf("rbx %X\n", r->mc_rbx); - runtime·printf("rcx %X\n", r->mc_rcx); - runtime·printf("rdx %X\n", r->mc_rdx); - runtime·printf("rdi %X\n", r->mc_rdi); - runtime·printf("rsi %X\n", r->mc_rsi); - runtime·printf("rbp %X\n", r->mc_rbp); - runtime·printf("rsp %X\n", r->mc_rsp); - runtime·printf("r8 %X\n", r->mc_r8 ); - runtime·printf("r9 %X\n", r->mc_r9 ); - runtime·printf("r10 %X\n", r->mc_r10); - runtime·printf("r11 %X\n", r->mc_r11); - runtime·printf("r12 %X\n", r->mc_r12); - runtime·printf("r13 %X\n", r->mc_r13); - runtime·printf("r14 %X\n", r->mc_r14); - runtime·printf("r15 %X\n", r->mc_r15); - runtime·printf("rip %X\n", r->mc_rip); - runtime·printf("rflags %X\n", r->mc_flags); - runtime·printf("cs %X\n", r->mc_cs); - runtime·printf("fs %X\n", r->mc_fs); - runtime·printf("gs %X\n", r->mc_gs); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Mcontext *r; - uintptr *sp; - SigTab *t; - - uc = context; - r = &uc->uc_mcontext; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->mc_rip, (uint8*)r->mc_rsp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = (uintptr)info->si_addr; - gp->sigpc = r->mc_rip; - - // Only push runtime·sigpanic if r->mc_rip != 0. - // If r->mc_rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->mc_rip != 0) { - sp = (uintptr*)r->mc_rsp; - *--sp = r->mc_rip; - r->mc_rsp = (uintptr)sp; - } - r->mc_rip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", r->mc_rip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->mc_rip, (void*)r->mc_rsp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = (int8*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa.__sigaction_u.__sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask.__bits[0] = ~(uint32)0; - sa.sa_mask.__bits[1] = ~(uint32)0; - sa.sa_mask.__bits[2] = ~(uint32)0; - sa.sa_mask.__bits[3] = ~(uint32)0; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} diff --git a/src/pkg/runtime/signal_freebsd_amd64.h b/src/pkg/runtime/signal_freebsd_amd64.h new file mode 100644 index 000000000..7d35b7f85 --- /dev/null +++ b/src/pkg/runtime/signal_freebsd_amd64.h @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) + +#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).mc_rax) +#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).mc_rbx) +#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).mc_rcx) +#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).mc_rdx) +#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).mc_rdi) +#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).mc_rsi) +#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).mc_rbp) +#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).mc_rsp) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).mc_r8) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).mc_r9) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).mc_r10) +#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).mc_r11) +#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).mc_r12) +#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).mc_r13) +#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).mc_r14) +#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).mc_r15) +#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).mc_rip) +#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).mc_rflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).mc_cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).mc_fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).mc_gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/pkg/runtime/signal_freebsd_arm.c b/src/pkg/runtime/signal_freebsd_arm.c deleted file mode 100644 index 50c3221bb..000000000 --- a/src/pkg/runtime/signal_freebsd_arm.c +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -#define r0 __gregs[0] -#define r1 __gregs[1] -#define r2 __gregs[2] -#define r3 __gregs[3] -#define r4 __gregs[4] -#define r5 __gregs[5] -#define r6 __gregs[6] -#define r7 __gregs[7] -#define r8 __gregs[8] -#define r9 __gregs[9] -#define r10 __gregs[10] -#define r11 __gregs[11] -#define r12 __gregs[12] -#define r13 __gregs[13] -#define r14 __gregs[14] -#define r15 __gregs[15] -#define cpsr __gregs[16] - -void -runtime·dumpregs(Mcontext *r) -{ - runtime·printf("r0 %x\n", r->r0); - runtime·printf("r1 %x\n", r->r1); - runtime·printf("r2 %x\n", r->r2); - runtime·printf("r3 %x\n", r->r3); - runtime·printf("r4 %x\n", r->r4); - runtime·printf("r5 %x\n", r->r5); - runtime·printf("r6 %x\n", r->r6); - runtime·printf("r7 %x\n", r->r7); - runtime·printf("r8 %x\n", r->r8); - runtime·printf("r9 %x\n", r->r9); - runtime·printf("r10 %x\n", r->r10); - runtime·printf("fp %x\n", r->r11); - runtime·printf("ip %x\n", r->r12); - runtime·printf("sp %x\n", r->r13); - runtime·printf("lr %x\n", r->r14); - runtime·printf("pc %x\n", r->r15); - runtime·printf("cpsr %x\n", r->cpsr); -} - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - int32 sa_flags; /* see signal options below */ - Sigset sa_mask; /* signal mask to apply */ -} Sigaction; - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Mcontext *r; - SigTab *t; - - uc = context; - r = &uc->uc_mcontext; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->r15, (uint8*)r->r13, (uint8*)r->r14, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = (uintptr)info->si_addr; - gp->sigpc = r->r15; - - // Only push runtime·sigpanic if r->mc_rip != 0. - // If r->mc_rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->r15 != 0) - r->r14 = r->r15; - // In case we are panicking from external C code - r->r10 = (uintptr)gp; - r->r9 = (uintptr)m; - r->r15 = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%x\n", r->r15); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->r15, (void*)r->r13, (void*)r->r14, gp); - runtime·tracebackothers(gp); - runtime·printf("\n"); - runtime·dumpregs(r); - } - -// breakpoint(); - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = (uint8*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa.__sigaction_u.__sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask.__bits[0] = ~(uint32)0; - sa.sa_mask.__bits[1] = ~(uint32)0; - sa.sa_mask.__bits[2] = ~(uint32)0; - sa.sa_mask.__bits[3] = ~(uint32)0; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -void -runtime·checkgoarm(void) -{ - // TODO(minux) -} - -#pragma textflag 7 -int64 -runtime·cputicks(void) -{ - // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). - // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. - // TODO: need more entropy to better seed fastrand1. - return runtime·nanotime(); -} diff --git a/src/pkg/runtime/signal_freebsd_arm.h b/src/pkg/runtime/signal_freebsd_arm.h new file mode 100644 index 000000000..87a45aa27 --- /dev/null +++ b/src/pkg/runtime/signal_freebsd_arm.h @@ -0,0 +1,28 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((Ucontext*)(ctxt))->uc_mcontext) + +#define SIG_R0(info, ctxt) (SIG_REGS(ctxt).__gregs[0]) +#define SIG_R1(info, ctxt) (SIG_REGS(ctxt).__gregs[1]) +#define SIG_R2(info, ctxt) (SIG_REGS(ctxt).__gregs[2]) +#define SIG_R3(info, ctxt) (SIG_REGS(ctxt).__gregs[3]) +#define SIG_R4(info, ctxt) (SIG_REGS(ctxt).__gregs[4]) +#define SIG_R5(info, ctxt) (SIG_REGS(ctxt).__gregs[5]) +#define SIG_R6(info, ctxt) (SIG_REGS(ctxt).__gregs[6]) +#define SIG_R7(info, ctxt) (SIG_REGS(ctxt).__gregs[7]) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).__gregs[8]) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).__gregs[9]) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).__gregs[10]) +#define SIG_FP(info, ctxt) (SIG_REGS(ctxt).__gregs[11]) +#define SIG_IP(info, ctxt) (SIG_REGS(ctxt).__gregs[12]) +#define SIG_SP(info, ctxt) (SIG_REGS(ctxt).__gregs[13]) +#define SIG_LR(info, ctxt) (SIG_REGS(ctxt).__gregs[14]) +#define SIG_PC(info, ctxt) (SIG_REGS(ctxt).__gregs[15]) +#define SIG_CPSR(info, ctxt) (SIG_REGS(ctxt).__gregs[16]) +#define SIG_FAULT(info, ctxt) ((uintptr)(info)->si_addr) +#define SIG_TRAP(info, ctxt) (0) +#define SIG_ERROR(info, ctxt) (0) +#define SIG_OLDMASK(info, ctxt) (0) +#define SIG_CODE0(info, ctxt) ((uintptr)(info)->si_code) diff --git a/src/pkg/runtime/signal_linux_386.c b/src/pkg/runtime/signal_linux_386.c deleted file mode 100644 index 9b45ec3bd..000000000 --- a/src/pkg/runtime/signal_linux_386.c +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -void -runtime·dumpregs(Sigcontext *r) -{ - runtime·printf("eax %x\n", r->eax); - runtime·printf("ebx %x\n", r->ebx); - runtime·printf("ecx %x\n", r->ecx); - runtime·printf("edx %x\n", r->edx); - runtime·printf("edi %x\n", r->edi); - runtime·printf("esi %x\n", r->esi); - runtime·printf("ebp %x\n", r->ebp); - runtime·printf("esp %x\n", r->esp); - runtime·printf("eip %x\n", r->eip); - runtime·printf("eflags %x\n", r->eflags); - runtime·printf("cs %x\n", r->cs); - runtime·printf("fs %x\n", r->fs); - runtime·printf("gs %x\n", r->gs); -} - -/* - * This assembler routine takes the args from registers, puts them on the stack, - * and calls sighandler(). - */ -extern void runtime·sigtramp(void); -extern void runtime·sigreturn(void); // calls runtime·sigreturn - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Sigcontext *r; - uintptr *sp; - SigTab *t; - - uc = context; - r = &uc->uc_mcontext; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->eip, (uint8*)r->esp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = ((uintptr*)info)[3]; - gp->sigpc = r->eip; - - // Only push runtime·sigpanic if r->eip != 0. - // If r->eip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->eip != 0) { - sp = (uintptr*)r->esp; - *--sp = r->eip; - r->esp = (uintptr)sp; - } - r->eip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", r->eip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->eip, (void*)r->esp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - if(runtime·rt_sigaction(i, nil, &sa, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction read failure"); - if(sa.k_sa_handler == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTORER; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - sa.sa_restorer = (void*)runtime·sigreturn; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.k_sa_handler = fn; - if(runtime·rt_sigaction(i, &sa, nil, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction failure"); -} - -#define AT_NULL 0 -#define AT_SYSINFO 32 -extern uint32 runtime·_vdso; - -#pragma textflag 7 -void -runtime·linux_setup_vdso(int32 argc, void *argv_list) -{ - byte **argv = &argv_list; - byte **envp; - uint32 *auxv; - - // skip envp to get to ELF auxiliary vector. - for(envp = &argv[argc+1]; *envp != nil; envp++) - ; - envp++; - - for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { - if(auxv[0] == AT_SYSINFO) { - runtime·_vdso = auxv[1]; - break; - } - } -} diff --git a/src/pkg/runtime/signal_linux_386.h b/src/pkg/runtime/signal_linux_386.h new file mode 100644 index 000000000..f77f1c9d5 --- /dev/null +++ b/src/pkg/runtime/signal_linux_386.h @@ -0,0 +1,24 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (*((Sigcontext*)&((Ucontext*)(ctxt))->uc_mcontext)) + +#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).eax) +#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).ebx) +#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).ecx) +#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).edx) +#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).edi) +#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).esi) +#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).ebp) +#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).esp) +#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).eip) +#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).eflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) (((uintptr*)(info))[2]) + diff --git a/src/pkg/runtime/signal_linux_amd64.c b/src/pkg/runtime/signal_linux_amd64.c deleted file mode 100644 index c4e39a6ab..000000000 --- a/src/pkg/runtime/signal_linux_amd64.c +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -void -runtime·dumpregs(Sigcontext *r) -{ - runtime·printf("rax %X\n", r->rax); - runtime·printf("rbx %X\n", r->rbx); - runtime·printf("rcx %X\n", r->rcx); - runtime·printf("rdx %X\n", r->rdx); - runtime·printf("rdi %X\n", r->rdi); - runtime·printf("rsi %X\n", r->rsi); - runtime·printf("rbp %X\n", r->rbp); - runtime·printf("rsp %X\n", r->rsp); - runtime·printf("r8 %X\n", r->r8 ); - runtime·printf("r9 %X\n", r->r9 ); - runtime·printf("r10 %X\n", r->r10); - runtime·printf("r11 %X\n", r->r11); - runtime·printf("r12 %X\n", r->r12); - runtime·printf("r13 %X\n", r->r13); - runtime·printf("r14 %X\n", r->r14); - runtime·printf("r15 %X\n", r->r15); - runtime·printf("rip %X\n", r->rip); - runtime·printf("rflags %X\n", r->eflags); - runtime·printf("cs %X\n", (uint64)r->cs); - runtime·printf("fs %X\n", (uint64)r->fs); - runtime·printf("gs %X\n", (uint64)r->gs); -} - -/* - * This assembler routine takes the args from registers, puts them on the stack, - * and calls sighandler(). - */ -extern void runtime·sigtramp(void); -extern void runtime·sigreturn(void); // calls runtime·sigreturn - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Mcontext *mc; - Sigcontext *r; - uintptr *sp; - SigTab *t; - - uc = context; - mc = &uc->uc_mcontext; - r = (Sigcontext*)mc; // same layout, more conveient names - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->rip, (uint8*)r->rsp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = ((uintptr*)info)[2]; - gp->sigpc = r->rip; - - // Only push runtime·sigpanic if r->rip != 0. - // If r->rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->rip != 0) { - sp = (uintptr*)r->rsp; - *--sp = r->rip; - r->rsp = (uintptr)sp; - } - r->rip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", r->rip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->rip, (void*)r->rsp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - if(runtime·rt_sigaction(i, nil, &sa, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction read failure"); - if(sa.sa_handler == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTORER; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - // TODO(adonovan): Linux manpage says "sa_restorer element is - // obsolete and should not be used". Avoid it here, and test. - sa.sa_restorer = (void*)runtime·sigreturn; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.sa_handler = fn; - if(runtime·rt_sigaction(i, &sa, nil, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction failure"); -} diff --git a/src/pkg/runtime/signal_linux_amd64.h b/src/pkg/runtime/signal_linux_amd64.h new file mode 100644 index 000000000..5a9a3e5da --- /dev/null +++ b/src/pkg/runtime/signal_linux_amd64.h @@ -0,0 +1,32 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (*((Sigcontext*)&((Ucontext*)(ctxt))->uc_mcontext)) + +#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).rax) +#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).rbx) +#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).rcx) +#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).rdx) +#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).rdi) +#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).rsi) +#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).rbp) +#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).rsp) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).r8) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).r9) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).r10) +#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).r11) +#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).r12) +#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).r13) +#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).r14) +#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).r15) +#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).rip) +#define SIG_RFLAGS(info, ctxt) ((uint64)SIG_REGS(ctxt).eflags) + +#define SIG_CS(info, ctxt) ((uint64)SIG_REGS(ctxt).cs) +#define SIG_FS(info, ctxt) ((uint64)SIG_REGS(ctxt).fs) +#define SIG_GS(info, ctxt) ((uint64)SIG_REGS(ctxt).gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) (((uintptr*)(info))[2]) + diff --git a/src/pkg/runtime/signal_linux_arm.c b/src/pkg/runtime/signal_linux_arm.c deleted file mode 100644 index c26caa7cd..000000000 --- a/src/pkg/runtime/signal_linux_arm.c +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -void -runtime·dumpregs(Sigcontext *r) -{ - runtime·printf("trap %x\n", r->trap_no); - runtime·printf("error %x\n", r->error_code); - runtime·printf("oldmask %x\n", r->oldmask); - runtime·printf("r0 %x\n", r->arm_r0); - runtime·printf("r1 %x\n", r->arm_r1); - runtime·printf("r2 %x\n", r->arm_r2); - runtime·printf("r3 %x\n", r->arm_r3); - runtime·printf("r4 %x\n", r->arm_r4); - runtime·printf("r5 %x\n", r->arm_r5); - runtime·printf("r6 %x\n", r->arm_r6); - runtime·printf("r7 %x\n", r->arm_r7); - runtime·printf("r8 %x\n", r->arm_r8); - runtime·printf("r9 %x\n", r->arm_r9); - runtime·printf("r10 %x\n", r->arm_r10); - runtime·printf("fp %x\n", r->arm_fp); - runtime·printf("ip %x\n", r->arm_ip); - runtime·printf("sp %x\n", r->arm_sp); - runtime·printf("lr %x\n", r->arm_lr); - runtime·printf("pc %x\n", r->arm_pc); - runtime·printf("cpsr %x\n", r->arm_cpsr); - runtime·printf("fault %x\n", r->fault_address); -} - -/* - * This assembler routine takes the args from registers, puts them on the stack, - * and calls sighandler(). - */ -extern void runtime·sigtramp(void); -extern void runtime·sigreturn(void); // calls runtime·sigreturn - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Ucontext *uc; - Sigcontext *r; - SigTab *t; - - uc = context; - r = &uc->uc_mcontext; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->arm_pc, (uint8*)r->arm_sp, (uint8*)r->arm_lr, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = r->fault_address; - gp->sigpc = r->arm_pc; - - // We arrange lr, and pc to pretend the panicking - // function calls sigpanic directly. - // Always save LR to stack so that panics in leaf - // functions are correctly handled. This smashes - // the stack frame but we're not going back there - // anyway. - r->arm_sp -= 4; - *(uint32 *)r->arm_sp = r->arm_lr; - // Don't bother saving PC if it's zero, which is - // probably a call to a nil func: the old link register - // is more useful in the stack trace. - if(r->arm_pc != 0) - r->arm_lr = r->arm_pc; - // In case we are panicking from external C code - r->arm_r10 = (uintptr)gp; - r->arm_r9 = (uintptr)m; - r->arm_pc = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - if(runtime·panicking) // traceback already printed - runtime·exit(2); - runtime·panicking = 1; - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%x\n", r->arm_pc); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->arm_pc, (void*)r->arm_sp, (void*)r->arm_lr, gp); - runtime·tracebackothers(gp); - runtime·printf("\n"); - runtime·dumpregs(r); - } - -// breakpoint(); - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - if(runtime·rt_sigaction(i, nil, &sa, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction read failure"); - if(sa.sa_handler == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_ONSTACK | SA_SIGINFO | SA_RESTORER; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - sa.sa_restorer = (void*)runtime·sigreturn; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.sa_handler = fn; - if(runtime·rt_sigaction(i, &sa, nil, sizeof(sa.sa_mask)) != 0) - runtime·throw("rt_sigaction failure"); -} - -#define AT_NULL 0 -#define AT_PLATFORM 15 // introduced in at least 2.6.11 -#define AT_HWCAP 16 // introduced in at least 2.6.11 -#define AT_RANDOM 25 // introduced in 2.6.29 -#define HWCAP_VFP (1 << 6) // introduced in at least 2.6.11 -#define HWCAP_VFPv3 (1 << 13) // introduced in 2.6.30 -static uint32 runtime·randomNumber; -uint8 runtime·armArch = 6; // we default to ARMv6 -uint32 runtime·hwcap; // set by setup_auxv -uint8 runtime·goarm; // set by 5l - -void -runtime·checkgoarm(void) -{ - if(runtime·goarm > 5 && !(runtime·hwcap & HWCAP_VFP)) { - runtime·printf("runtime: this CPU has no floating point hardware, so it cannot run\n"); - runtime·printf("this GOARM=%d binary. Recompile using GOARM=5.\n", runtime·goarm); - runtime·exit(1); - } - if(runtime·goarm > 6 && !(runtime·hwcap & HWCAP_VFPv3)) { - runtime·printf("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n"); - runtime·printf("this GOARM=%d binary. Recompile using GOARM=6.\n", runtime·goarm); - runtime·exit(1); - } -} - -#pragma textflag 7 -void -runtime·setup_auxv(int32 argc, void *argv_list) -{ - byte **argv; - byte **envp; - byte *rnd; - uint32 *auxv; - uint32 t; - - argv = &argv_list; - - // skip envp to get to ELF auxiliary vector. - for(envp = &argv[argc+1]; *envp != nil; envp++) - ; - envp++; - - for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) { - switch(auxv[0]) { - case AT_RANDOM: // kernel provided 16-byte worth of random data - if(auxv[1]) { - rnd = (byte*)auxv[1]; - runtime·randomNumber = rnd[4] | rnd[5]<<8 | rnd[6]<<16 | rnd[7]<<24; - } - break; - case AT_PLATFORM: // v5l, v6l, v7l - if(auxv[1]) { - t = *(uint8*)(auxv[1]+1); - if(t >= '5' && t <= '7') - runtime·armArch = t - '0'; - } - break; - case AT_HWCAP: // CPU capability bit flags - runtime·hwcap = auxv[1]; - break; - } - } -} - -#pragma textflag 7 -int64 -runtime·cputicks(void) -{ - // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). - // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. - // runtime·randomNumber provides better seeding of fastrand1. - return runtime·nanotime() + runtime·randomNumber; -} diff --git a/src/pkg/runtime/signal_linux_arm.h b/src/pkg/runtime/signal_linux_arm.h new file mode 100644 index 000000000..a674c0d57 --- /dev/null +++ b/src/pkg/runtime/signal_linux_arm.h @@ -0,0 +1,28 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (*((Sigcontext*)&((Ucontext*)(ctxt))->uc_mcontext)) + +#define SIG_R0(info, ctxt) (SIG_REGS(ctxt).arm_r0) +#define SIG_R1(info, ctxt) (SIG_REGS(ctxt).arm_r1) +#define SIG_R2(info, ctxt) (SIG_REGS(ctxt).arm_r2) +#define SIG_R3(info, ctxt) (SIG_REGS(ctxt).arm_r3) +#define SIG_R4(info, ctxt) (SIG_REGS(ctxt).arm_r4) +#define SIG_R5(info, ctxt) (SIG_REGS(ctxt).arm_r5) +#define SIG_R6(info, ctxt) (SIG_REGS(ctxt).arm_r6) +#define SIG_R7(info, ctxt) (SIG_REGS(ctxt).arm_r7) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).arm_r8) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).arm_r9) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).arm_r10) +#define SIG_FP(info, ctxt) (SIG_REGS(ctxt).arm_fp) +#define SIG_IP(info, ctxt) (SIG_REGS(ctxt).arm_ip) +#define SIG_SP(info, ctxt) (SIG_REGS(ctxt).arm_sp) +#define SIG_LR(info, ctxt) (SIG_REGS(ctxt).arm_lr) +#define SIG_PC(info, ctxt) (SIG_REGS(ctxt).arm_pc) +#define SIG_CPSR(info, ctxt) (SIG_REGS(ctxt).arm_cpsr) +#define SIG_FAULT(info, ctxt) (SIG_REGS(ctxt).fault_address) +#define SIG_TRAP(info, ctxt) (SIG_REGS(ctxt).trap_no) +#define SIG_ERROR(info, ctxt) (SIG_REGS(ctxt).error_code) +#define SIG_OLDMASK(info, ctxt) (SIG_REGS(ctxt).oldmask) +#define SIG_CODE0(info, ctxt) ((uintptr)(info)->si_code) diff --git a/src/pkg/runtime/signal_netbsd_386.c b/src/pkg/runtime/signal_netbsd_386.c deleted file mode 100644 index 08744c425..000000000 --- a/src/pkg/runtime/signal_netbsd_386.c +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -extern void runtime·lwp_tramp(void); -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*_sa_handler)(int32); - void (*_sa_sigaction)(int32, Siginfo*, void *); - } _sa_u; /* signal handler */ - uint32 sa_mask[4]; /* signal mask to apply */ - int32 sa_flags; /* see signal options below */ -} Sigaction; - -void -runtime·dumpregs(McontextT *mc) -{ - runtime·printf("eax %x\n", mc->__gregs[REG_EAX]); - runtime·printf("ebx %x\n", mc->__gregs[REG_EBX]); - runtime·printf("ecx %x\n", mc->__gregs[REG_ECX]); - runtime·printf("edx %x\n", mc->__gregs[REG_EDX]); - runtime·printf("edi %x\n", mc->__gregs[REG_EDI]); - runtime·printf("esi %x\n", mc->__gregs[REG_ESI]); - runtime·printf("ebp %x\n", mc->__gregs[REG_EBP]); - runtime·printf("esp %x\n", mc->__gregs[REG_UESP]); - runtime·printf("eip %x\n", mc->__gregs[REG_EIP]); - runtime·printf("eflags %x\n", mc->__gregs[REG_EFL]); - runtime·printf("cs %x\n", mc->__gregs[REG_CS]); - runtime·printf("fs %x\n", mc->__gregs[REG_FS]); - runtime·printf("gs %x\n", mc->__gregs[REG_GS]); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - UcontextT *uc = context; - McontextT *mc = &uc->uc_mcontext; - uintptr *sp; - SigTab *t; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)mc->__gregs[REG_EIP], - (uint8*)mc->__gregs[REG_UESP], nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // We need to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->_code; - gp->sigcode1 = *(uintptr*)&info->_reason[0]; /* _addr */ - gp->sigpc = mc->__gregs[REG_EIP]; - - // Only push runtime·sigpanic if __gregs[REG_EIP] != 0. - // If __gregs[REG_EIP] == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will make the - // trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic - // and we won't get to see who faulted.) - if(mc->__gregs[REG_EIP] != 0) { - sp = (uintptr*)mc->__gregs[REG_UESP]; - *--sp = mc->__gregs[REG_EIP]; - mc->__gregs[REG_UESP] = (uintptr)sp; - } - mc->__gregs[REG_EIP] = (uintptr)runtime·sigpanic; - return; - } - - if(info->_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", mc->__gregs[REG_EIP]); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)mc->__gregs[REG_EIP], - (void*)mc->__gregs[REG_UESP], 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(mc); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa._sa_u._sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask[0] = ~0U; - sa.sa_mask[1] = ~0U; - sa.sa_mask[2] = ~0U; - sa.sa_mask[3] = ~0U; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa._sa_u._sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -void -runtime·lwp_mcontext_init(McontextT *mc, void *stack, M *mp, G *gp, void (*fn)(void)) -{ - mc->__gregs[REG_EIP] = (uint32)runtime·lwp_tramp; - mc->__gregs[REG_UESP] = (uint32)stack; - mc->__gregs[REG_EBX] = (uint32)mp; - mc->__gregs[REG_EDX] = (uint32)gp; - mc->__gregs[REG_ESI] = (uint32)fn; -} diff --git a/src/pkg/runtime/signal_netbsd_386.h b/src/pkg/runtime/signal_netbsd_386.h new file mode 100644 index 000000000..d5a8a0c4b --- /dev/null +++ b/src/pkg/runtime/signal_netbsd_386.h @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((UcontextT*)(ctxt))->uc_mcontext) + +#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EAX]) +#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EBX]) +#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_ECX]) +#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EDX]) +#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EDI]) +#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_ESI]) +#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EBP]) +#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_UESP]) +#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EIP]) +#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_EFL]) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_CS]) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_FS]) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_GS]) + +#define SIG_CODE0(info, ctxt) ((info)->_code) +#define SIG_CODE1(info, ctxt) (*(uintptr*)&(info)->_reason[0]) diff --git a/src/pkg/runtime/signal_netbsd_amd64.c b/src/pkg/runtime/signal_netbsd_amd64.c deleted file mode 100644 index 46afb682b..000000000 --- a/src/pkg/runtime/signal_netbsd_amd64.c +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -extern void runtime·lwp_tramp(void); -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*_sa_handler)(int32); - void (*_sa_sigaction)(int32, Siginfo*, void *); - } _sa_u; /* signal handler */ - uint32 sa_mask[4]; /* signal mask to apply */ - int32 sa_flags; /* see signal options below */ -} Sigaction; - -void -runtime·dumpregs(McontextT *mc) -{ - runtime·printf("rax %X\n", mc->__gregs[REG_RAX]); - runtime·printf("rbx %X\n", mc->__gregs[REG_RBX]); - runtime·printf("rcx %X\n", mc->__gregs[REG_RCX]); - runtime·printf("rdx %X\n", mc->__gregs[REG_RDX]); - runtime·printf("rdi %X\n", mc->__gregs[REG_RDI]); - runtime·printf("rsi %X\n", mc->__gregs[REG_RSI]); - runtime·printf("rbp %X\n", mc->__gregs[REG_RBP]); - runtime·printf("rsp %X\n", mc->__gregs[REG_RSP]); - runtime·printf("r8 %X\n", mc->__gregs[REG_R8]); - runtime·printf("r9 %X\n", mc->__gregs[REG_R9]); - runtime·printf("r10 %X\n", mc->__gregs[REG_R10]); - runtime·printf("r11 %X\n", mc->__gregs[REG_R11]); - runtime·printf("r12 %X\n", mc->__gregs[REG_R12]); - runtime·printf("r13 %X\n", mc->__gregs[REG_R13]); - runtime·printf("r14 %X\n", mc->__gregs[REG_R14]); - runtime·printf("r15 %X\n", mc->__gregs[REG_R15]); - runtime·printf("rip %X\n", mc->__gregs[REG_RIP]); - runtime·printf("rflags %X\n", mc->__gregs[REG_RFLAGS]); - runtime·printf("cs %X\n", mc->__gregs[REG_CS]); - runtime·printf("fs %X\n", mc->__gregs[REG_FS]); - runtime·printf("gs %X\n", mc->__gregs[REG_GS]); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - UcontextT *uc = context; - McontextT *mc = &uc->uc_mcontext; - uintptr *sp; - SigTab *t; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)mc->__gregs[REG_RIP], - (uint8*)mc->__gregs[REG_RSP], nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // We need to pass arguments out of band since augmenting the - // stack frame would break the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->_code; - gp->sigcode1 = *(uintptr*)&info->_reason[0]; /* _addr */ - gp->sigpc = mc->__gregs[REG_RIP]; - - // Only push runtime·sigpanic if __gregs[REG_RIP] != 0. - // If __gregs[REG_RIP] == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will make the - // trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic - // and we won't get to see who faulted.) - if(mc->__gregs[REG_RIP] != 0) { - sp = (uintptr*)mc->__gregs[REG_RSP]; - *--sp = mc->__gregs[REG_RIP]; - mc->__gregs[REG_RSP] = (uintptr)sp; - } - mc->__gregs[REG_RIP] = (uintptr)runtime·sigpanic; - return; - } - - if(info->_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", mc->__gregs[REG_RIP]); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)mc->__gregs[REG_RIP], - (void*)mc->__gregs[REG_RSP], 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(mc); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa._sa_u._sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask[0] = ~0U; - sa.sa_mask[1] = ~0U; - sa.sa_mask[2] = ~0U; - sa.sa_mask[3] = ~0U; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa._sa_u._sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -void -runtime·lwp_mcontext_init(McontextT *mc, void *stack, M *mp, G *gp, void (*fn)(void)) -{ - // Machine dependent mcontext initialisation for LWP. - mc->__gregs[REG_RIP] = (uint64)runtime·lwp_tramp; - mc->__gregs[REG_RSP] = (uint64)stack; - mc->__gregs[REG_R8] = (uint64)mp; - mc->__gregs[REG_R9] = (uint64)gp; - mc->__gregs[REG_R12] = (uint64)fn; -} diff --git a/src/pkg/runtime/signal_netbsd_amd64.h b/src/pkg/runtime/signal_netbsd_amd64.h new file mode 100644 index 000000000..7ec4cd98c --- /dev/null +++ b/src/pkg/runtime/signal_netbsd_amd64.h @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((UcontextT*)(ctxt))->uc_mcontext) + +#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RAX]) +#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RBX]) +#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RCX]) +#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RDX]) +#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RDI]) +#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RSI]) +#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RBP]) +#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RSP]) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R8]) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R9]) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R10]) +#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R11]) +#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R12]) +#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R13]) +#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R14]) +#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R15]) +#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RIP]) +#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_RFLAGS]) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_CS]) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_FS]) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_GS]) + +#define SIG_CODE0(info, ctxt) ((info)->_code) +#define SIG_CODE1(info, ctxt) (*(uintptr*)&(info)->_reason[0]) diff --git a/src/pkg/runtime/signal_netbsd_arm.c b/src/pkg/runtime/signal_netbsd_arm.c deleted file mode 100644 index 97f62687b..000000000 --- a/src/pkg/runtime/signal_netbsd_arm.c +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -#define r0 __gregs[0] -#define r1 __gregs[1] -#define r2 __gregs[2] -#define r3 __gregs[3] -#define r4 __gregs[4] -#define r5 __gregs[5] -#define r6 __gregs[6] -#define r7 __gregs[7] -#define r8 __gregs[8] -#define r9 __gregs[9] -#define r10 __gregs[10] -#define r11 __gregs[11] -#define r12 __gregs[12] -#define r13 __gregs[13] -#define r14 __gregs[14] -#define r15 __gregs[15] -#define cpsr __gregs[16] - -void -runtime·dumpregs(McontextT *r) -{ - runtime·printf("r0 %x\n", r->r0); - runtime·printf("r1 %x\n", r->r1); - runtime·printf("r2 %x\n", r->r2); - runtime·printf("r3 %x\n", r->r3); - runtime·printf("r4 %x\n", r->r4); - runtime·printf("r5 %x\n", r->r5); - runtime·printf("r6 %x\n", r->r6); - runtime·printf("r7 %x\n", r->r7); - runtime·printf("r8 %x\n", r->r8); - runtime·printf("r9 %x\n", r->r9); - runtime·printf("r10 %x\n", r->r10); - runtime·printf("fp %x\n", r->r11); - runtime·printf("ip %x\n", r->r12); - runtime·printf("sp %x\n", r->r13); - runtime·printf("lr %x\n", r->r14); - runtime·printf("pc %x\n", r->r15); - runtime·printf("cpsr %x\n", r->cpsr); -} - -extern void runtime·lwp_tramp(void); -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*_sa_handler)(int32); - void (*_sa_sigaction)(int32, Siginfo*, void *); - } _sa_u; /* signal handler */ - uint32 sa_mask[4]; /* signal mask to apply */ - int32 sa_flags; /* see signal options below */ -} Sigaction; - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - UcontextT *uc; - McontextT *r; - SigTab *t; - - uc = context; - r = &uc->uc_mcontext; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->r15, (uint8*)r->r13, (uint8*)r->r14, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // We have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->_code; - gp->sigcode1 = *(uintptr*)&info->_reason[0]; /* _addr */ - gp->sigpc = r->r15; - - // We arrange lr, and pc to pretend the panicking - // function calls sigpanic directly. - // Always save LR to stack so that panics in leaf - // functions are correctly handled. This smashes - // the stack frame but we're not going back there - // anyway. - r->r13 -= 4; - *(uint32 *)r->r13 = r->r14; - // Don't bother saving PC if it's zero, which is - // probably a call to a nil func: the old link register - // is more useful in the stack trace. - if(r->r15 != 0) - r->r14 = r->r15; - // In case we are panicking from external C code - r->r10 = (uintptr)gp; - r->r9 = (uintptr)m; - r->r15 = (uintptr)runtime·sigpanic; - return; - } - - if(info->_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%x\n", r->r15); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->r15, (void*)r->r13, (void*)r->r14, gp); - runtime·tracebackothers(gp); - runtime·printf("\n"); - runtime·dumpregs(r); - } - -// breakpoint(); - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = (uint8*)p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa._sa_u._sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask[0] = ~0U; - sa.sa_mask[1] = ~0U; - sa.sa_mask[2] = ~0U; - sa.sa_mask[3] = ~0U; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa._sa_u._sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} - -void -runtime·lwp_mcontext_init(McontextT *mc, void *stack, M *mp, G *gp, void (*fn)(void)) -{ - mc->r15 = (uint32)runtime·lwp_tramp; - mc->r13 = (uint32)stack; - mc->r0 = (uint32)mp; - mc->r1 = (uint32)gp; - mc->r2 = (uint32)fn; -} - -void -runtime·checkgoarm(void) -{ - // TODO(minux) -} - -#pragma textflag 7 -int64 -runtime·cputicks() { - // Currently cputicks() is used in blocking profiler and to seed runtime·fastrand1(). - // runtime·nanotime() is a poor approximation of CPU ticks that is enough for the profiler. - // TODO: need more entropy to better seed fastrand1. - return runtime·nanotime(); -} diff --git a/src/pkg/runtime/signal_netbsd_arm.h b/src/pkg/runtime/signal_netbsd_arm.h new file mode 100644 index 000000000..12f5827a6 --- /dev/null +++ b/src/pkg/runtime/signal_netbsd_arm.h @@ -0,0 +1,30 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (((UcontextT*)(ctxt))->uc_mcontext) + +#define SIG_R0(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R0]) +#define SIG_R1(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R1]) +#define SIG_R2(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R2]) +#define SIG_R3(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R3]) +#define SIG_R4(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R4]) +#define SIG_R5(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R5]) +#define SIG_R6(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R6]) +#define SIG_R7(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R7]) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R8]) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R9]) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R10]) +#define SIG_FP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R11]) +#define SIG_IP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R12]) +#define SIG_SP(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R13]) +#define SIG_LR(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R14]) +#define SIG_PC(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_R15]) +#define SIG_CPSR(info, ctxt) (SIG_REGS(ctxt).__gregs[REG_CPSR]) +#define SIG_FAULT(info, ctxt) (*(uintptr*)&(info)->_reason[0]) +#define SIG_TRAP(info, ctxt) (0) +#define SIG_ERROR(info, ctxt) (0) +#define SIG_OLDMASK(info, ctxt) (0) + +#define SIG_CODE0(info, ctxt) ((info)->_code) +#define SIG_CODE1(info, ctxt) (*(uintptr*)&(info)->_reason[0]) diff --git a/src/pkg/runtime/signal_openbsd_386.c b/src/pkg/runtime/signal_openbsd_386.c deleted file mode 100644 index 516797c8d..000000000 --- a/src/pkg/runtime/signal_openbsd_386.c +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - uint32 sa_mask; /* signal mask to apply */ - int32 sa_flags; /* see signal options below */ -} Sigaction; - -void -runtime·dumpregs(Sigcontext *r) -{ - runtime·printf("eax %x\n", r->sc_eax); - runtime·printf("ebx %x\n", r->sc_ebx); - runtime·printf("ecx %x\n", r->sc_ecx); - runtime·printf("edx %x\n", r->sc_edx); - runtime·printf("edi %x\n", r->sc_edi); - runtime·printf("esi %x\n", r->sc_esi); - runtime·printf("ebp %x\n", r->sc_ebp); - runtime·printf("esp %x\n", r->sc_esp); - runtime·printf("eip %x\n", r->sc_eip); - runtime·printf("eflags %x\n", r->sc_eflags); - runtime·printf("cs %x\n", r->sc_cs); - runtime·printf("fs %x\n", r->sc_fs); - runtime·printf("gs %x\n", r->sc_gs); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Sigcontext *r = context; - uintptr *sp; - SigTab *t; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->sc_eip, (uint8*)r->sc_esp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = *(uintptr*)((byte*)info + 12); /* si_addr */ - gp->sigpc = r->sc_eip; - - // Only push runtime·sigpanic if r->sc_eip != 0. - // If r->sc_eip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->sc_eip != 0) { - sp = (uintptr*)r->sc_esp; - *--sp = r->sc_eip; - r->sc_esp = (uintptr)sp; - } - r->sc_eip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", r->sc_eip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->sc_eip, (void*)r->sc_esp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa.__sigaction_u.__sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0ULL; - if (fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} diff --git a/src/pkg/runtime/signal_openbsd_386.h b/src/pkg/runtime/signal_openbsd_386.h new file mode 100644 index 000000000..0ba66ab9f --- /dev/null +++ b/src/pkg/runtime/signal_openbsd_386.h @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (*(Sigcontext*)(ctxt)) + +#define SIG_EAX(info, ctxt) (SIG_REGS(ctxt).sc_eax) +#define SIG_EBX(info, ctxt) (SIG_REGS(ctxt).sc_ebx) +#define SIG_ECX(info, ctxt) (SIG_REGS(ctxt).sc_ecx) +#define SIG_EDX(info, ctxt) (SIG_REGS(ctxt).sc_edx) +#define SIG_EDI(info, ctxt) (SIG_REGS(ctxt).sc_edi) +#define SIG_ESI(info, ctxt) (SIG_REGS(ctxt).sc_esi) +#define SIG_EBP(info, ctxt) (SIG_REGS(ctxt).sc_ebp) +#define SIG_ESP(info, ctxt) (SIG_REGS(ctxt).sc_esp) +#define SIG_EIP(info, ctxt) (SIG_REGS(ctxt).sc_eip) +#define SIG_EFLAGS(info, ctxt) (SIG_REGS(ctxt).sc_eflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).sc_cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).sc_fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).sc_gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) ((uintptr)(info)->si_addr) diff --git a/src/pkg/runtime/signal_openbsd_amd64.c b/src/pkg/runtime/signal_openbsd_amd64.c deleted file mode 100644 index 0d0db770b..000000000 --- a/src/pkg/runtime/signal_openbsd_amd64.c +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "runtime.h" -#include "defs_GOOS_GOARCH.h" -#include "signals_GOOS.h" -#include "os_GOOS.h" - -extern void runtime·sigtramp(void); - -typedef struct sigaction { - union { - void (*__sa_handler)(int32); - void (*__sa_sigaction)(int32, Siginfo*, void *); - } __sigaction_u; /* signal handler */ - uint32 sa_mask; /* signal mask to apply */ - int32 sa_flags; /* see signal options below */ -} Sigaction; - -void -runtime·dumpregs(Sigcontext *r) -{ - runtime·printf("rax %X\n", r->sc_rax); - runtime·printf("rbx %X\n", r->sc_rbx); - runtime·printf("rcx %X\n", r->sc_rcx); - runtime·printf("rdx %X\n", r->sc_rdx); - runtime·printf("rdi %X\n", r->sc_rdi); - runtime·printf("rsi %X\n", r->sc_rsi); - runtime·printf("rbp %X\n", r->sc_rbp); - runtime·printf("rsp %X\n", r->sc_rsp); - runtime·printf("r8 %X\n", r->sc_r8); - runtime·printf("r9 %X\n", r->sc_r9); - runtime·printf("r10 %X\n", r->sc_r10); - runtime·printf("r11 %X\n", r->sc_r11); - runtime·printf("r12 %X\n", r->sc_r12); - runtime·printf("r13 %X\n", r->sc_r13); - runtime·printf("r14 %X\n", r->sc_r14); - runtime·printf("r15 %X\n", r->sc_r15); - runtime·printf("rip %X\n", r->sc_rip); - runtime·printf("rflags %X\n", r->sc_rflags); - runtime·printf("cs %X\n", r->sc_cs); - runtime·printf("fs %X\n", r->sc_fs); - runtime·printf("gs %X\n", r->sc_gs); -} - -void -runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp) -{ - Sigcontext *r = context; - uintptr *sp; - SigTab *t; - - if(sig == SIGPROF) { - runtime·sigprof((uint8*)r->sc_rip, - (uint8*)r->sc_rsp, nil, gp); - return; - } - - t = &runtime·sigtab[sig]; - if(info->si_code != SI_USER && (t->flags & SigPanic)) { - if(gp == nil || gp == m->g0) - goto Throw; - // Make it look like a call to the signal func. - // Have to pass arguments out of band since - // augmenting the stack frame would break - // the unwinding code. - gp->sig = sig; - gp->sigcode0 = info->si_code; - gp->sigcode1 = *(uintptr*)((byte*)info + 16); /* si_addr */ - gp->sigpc = r->sc_rip; - - // Only push runtime·sigpanic if r->sc_rip != 0. - // If r->sc_rip == 0, probably panicked because of a - // call to a nil func. Not pushing that onto sp will - // make the trace look like a call to runtime·sigpanic instead. - // (Otherwise the trace will end at runtime·sigpanic and we - // won't get to see who faulted.) - if(r->sc_rip != 0) { - sp = (uintptr*)r->sc_rsp; - *--sp = r->sc_rip; - r->sc_rsp = (uintptr)sp; - } - r->sc_rip = (uintptr)runtime·sigpanic; - return; - } - - if(info->si_code == SI_USER || (t->flags & SigNotify)) - if(runtime·sigsend(sig)) - return; - if(t->flags & SigKill) - runtime·exit(2); - if(!(t->flags & SigThrow)) - return; - -Throw: - runtime·startpanic(); - - if(sig < 0 || sig >= NSIG) - runtime·printf("Signal %d\n", sig); - else - runtime·printf("%s\n", runtime·sigtab[sig].name); - - runtime·printf("PC=%X\n", r->sc_rip); - if(m->lockedg != nil && m->ncgo > 0 && gp == m->g0) { - runtime·printf("signal arrived during cgo execution\n"); - gp = m->lockedg; - } - runtime·printf("\n"); - - if(runtime·gotraceback()){ - runtime·traceback((void*)r->sc_rip, (void*)r->sc_rsp, 0, gp); - runtime·tracebackothers(gp); - runtime·dumpregs(r); - } - - runtime·exit(2); -} - -void -runtime·signalstack(byte *p, int32 n) -{ - Sigaltstack st; - - st.ss_sp = p; - st.ss_size = n; - st.ss_flags = 0; - if(p == nil) - st.ss_flags = SS_DISABLE; - runtime·sigaltstack(&st, nil); -} - -void -runtime·setsig(int32 i, void (*fn)(int32, Siginfo*, void*, G*), bool restart) -{ - Sigaction sa; - - // If SIGHUP handler is SIG_IGN, assume running - // under nohup and do not set explicit handler. - if(i == SIGHUP) { - runtime·memclr((byte*)&sa, sizeof sa); - runtime·sigaction(i, nil, &sa); - if(sa.__sigaction_u.__sa_sigaction == SIG_IGN) - return; - } - - runtime·memclr((byte*)&sa, sizeof sa); - sa.sa_flags = SA_SIGINFO|SA_ONSTACK; - if(restart) - sa.sa_flags |= SA_RESTART; - sa.sa_mask = ~0U; - if(fn == runtime·sighandler) - fn = (void*)runtime·sigtramp; - sa.__sigaction_u.__sa_sigaction = (void*)fn; - runtime·sigaction(i, &sa, nil); -} diff --git a/src/pkg/runtime/signal_openbsd_amd64.h b/src/pkg/runtime/signal_openbsd_amd64.h new file mode 100644 index 000000000..b46a5dfa6 --- /dev/null +++ b/src/pkg/runtime/signal_openbsd_amd64.h @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_REGS(ctxt) (*(Sigcontext*)(ctxt)) + +#define SIG_RAX(info, ctxt) (SIG_REGS(ctxt).sc_rax) +#define SIG_RBX(info, ctxt) (SIG_REGS(ctxt).sc_rbx) +#define SIG_RCX(info, ctxt) (SIG_REGS(ctxt).sc_rcx) +#define SIG_RDX(info, ctxt) (SIG_REGS(ctxt).sc_rdx) +#define SIG_RDI(info, ctxt) (SIG_REGS(ctxt).sc_rdi) +#define SIG_RSI(info, ctxt) (SIG_REGS(ctxt).sc_rsi) +#define SIG_RBP(info, ctxt) (SIG_REGS(ctxt).sc_rbp) +#define SIG_RSP(info, ctxt) (SIG_REGS(ctxt).sc_rsp) +#define SIG_R8(info, ctxt) (SIG_REGS(ctxt).sc_r8) +#define SIG_R9(info, ctxt) (SIG_REGS(ctxt).sc_r9) +#define SIG_R10(info, ctxt) (SIG_REGS(ctxt).sc_r10) +#define SIG_R11(info, ctxt) (SIG_REGS(ctxt).sc_r11) +#define SIG_R12(info, ctxt) (SIG_REGS(ctxt).sc_r12) +#define SIG_R13(info, ctxt) (SIG_REGS(ctxt).sc_r13) +#define SIG_R14(info, ctxt) (SIG_REGS(ctxt).sc_r14) +#define SIG_R15(info, ctxt) (SIG_REGS(ctxt).sc_r15) +#define SIG_RIP(info, ctxt) (SIG_REGS(ctxt).sc_rip) +#define SIG_RFLAGS(info, ctxt) (SIG_REGS(ctxt).sc_rflags) + +#define SIG_CS(info, ctxt) (SIG_REGS(ctxt).sc_cs) +#define SIG_FS(info, ctxt) (SIG_REGS(ctxt).sc_fs) +#define SIG_GS(info, ctxt) (SIG_REGS(ctxt).sc_gs) + +#define SIG_CODE0(info, ctxt) ((info)->si_code) +#define SIG_CODE1(info, ctxt) (*(uintptr*)((byte*)(info) + 16)) diff --git a/src/pkg/runtime/signal_unix.c b/src/pkg/runtime/signal_unix.c index 9b7e8b03a..54e461f99 100644 --- a/src/pkg/runtime/signal_unix.c +++ b/src/pkg/runtime/signal_unix.c @@ -7,6 +7,7 @@ #include "runtime.h" #include "defs_GOOS_GOARCH.h" #include "os_GOOS.h" +#include "signal_unix.h" extern SigTab runtime·sigtab[]; @@ -21,6 +22,20 @@ runtime·initsig(void) t = &runtime·sigtab[i]; if((t->flags == 0) || (t->flags & SigDefault)) continue; + + // For some signals, we respect an inherited SIG_IGN handler + // rather than insist on installing our own default handler. + // Even these signals can be fetched using the os/signal package. + switch(i) { + case SIGHUP: + case SIGINT: + if(runtime·getsig(i) == SIG_IGN) { + t->flags = SigNotify | SigIgnored; + continue; + } + } + + t->flags |= SigHandling; runtime·setsig(i, runtime·sighandler, true); } } @@ -28,18 +43,35 @@ runtime·initsig(void) void runtime·sigenable(uint32 sig) { - int32 i; SigTab *t; - for(i = 0; i<NSIG; i++) { - // ~0 means all signals. - if(~sig == 0 || i == sig) { - t = &runtime·sigtab[i]; - if(t->flags & SigDefault) { - runtime·setsig(i, runtime·sighandler, true); - t->flags &= ~SigDefault; // make this idempotent - } - } + if(sig >= NSIG) + return; + + t = &runtime·sigtab[sig]; + if((t->flags & SigNotify) && !(t->flags & SigHandling)) { + t->flags |= SigHandling; + if(runtime·getsig(sig) == SIG_IGN) + t->flags |= SigIgnored; + runtime·setsig(sig, runtime·sighandler, true); + } +} + +void +runtime·sigdisable(uint32 sig) +{ + SigTab *t; + + if(sig >= NSIG) + return; + + t = &runtime·sigtab[sig]; + if((t->flags & SigNotify) && (t->flags & SigHandling)) { + t->flags &= ~SigHandling; + if(t->flags & SigIgnored) + runtime·setsig(sig, SIG_IGN, true); + else + runtime·setsig(sig, SIG_DFL, true); } } @@ -66,5 +98,23 @@ void os·sigpipe(void) { runtime·setsig(SIGPIPE, SIG_DFL, false); - runtime·raisesigpipe(); + runtime·raise(SIGPIPE); +} + +void +runtime·crash(void) +{ +#ifdef GOOS_darwin + // OS X core dumps are linear dumps of the mapped memory, + // from the first virtual byte to the last, with zeros in the gaps. + // Because of the way we arrange the address space on 64-bit systems, + // this means the OS X core file will be >128 GB and even on a zippy + // workstation can take OS X well over an hour to write (uninterruptible). + // Save users from making that mistake. + if(sizeof(void*) == 8) + return; +#endif + + runtime·setsig(SIGABRT, SIG_DFL, false); + runtime·raise(SIGABRT); } diff --git a/src/pkg/runtime/signal_unix.h b/src/pkg/runtime/signal_unix.h new file mode 100644 index 000000000..2d84a0186 --- /dev/null +++ b/src/pkg/runtime/signal_unix.h @@ -0,0 +1,14 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#define SIG_DFL ((void*)0) +#define SIG_IGN ((void*)1) + +typedef void GoSighandler(int32, Siginfo*, void*, G*); +void runtime·setsig(int32, GoSighandler*, bool); +GoSighandler* runtime·getsig(int32); + +void runtime·sighandler(int32 sig, Siginfo *info, void *context, G *gp); +void runtime·raise(int32); + diff --git a/src/pkg/runtime/signals_plan9.h b/src/pkg/runtime/signals_plan9.h index 0f1165e2a..f9bec65fc 100644 --- a/src/pkg/runtime/signals_plan9.h +++ b/src/pkg/runtime/signals_plan9.h @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + #define N SigNotify #define T SigThrow #define P SigPanic diff --git a/src/pkg/runtime/sigqueue.goc b/src/pkg/runtime/sigqueue.goc index ab5f312e4..7e083685d 100644 --- a/src/pkg/runtime/sigqueue.goc +++ b/src/pkg/runtime/sigqueue.goc @@ -133,8 +133,6 @@ done:; // Must only be called from a single goroutine at a time. func signal_enable(s uint32) { - int32 i; - if(!sig.inuse) { // The first call to signal_enable is for us // to use for initialization. It does not pass @@ -144,16 +142,16 @@ func signal_enable(s uint32) { return; } - if(~s == 0) { - // Special case: want everything. - for(i=0; i<nelem(sig.wanted); i++) - sig.wanted[i] = ~(uint32)0; - runtime·sigenable(s); - return; - } - if(s >= nelem(sig.wanted)*32) return; sig.wanted[s/32] |= 1U<<(s&31); runtime·sigenable(s); } + +// Must only be called from a single goroutine at a time. +func signal_disable(s uint32) { + if(s >= nelem(sig.wanted)*32) + return; + sig.wanted[s/32] &= ~(1U<<(s&31)); + runtime·sigdisable(s); +} diff --git a/src/pkg/runtime/stack_test.go b/src/pkg/runtime/stack_test.go index 759f7c46e..da0181a66 100644 --- a/src/pkg/runtime/stack_test.go +++ b/src/pkg/runtime/stack_test.go @@ -1533,7 +1533,7 @@ func stack5000() (uintptr, uintptr) { var buf [5000]byte; use(buf[:]); return St func TestStackMem(t *testing.T) { const ( BatchSize = 32 - BatchCount = 512 + BatchCount = 256 ArraySize = 1024 RecursionDepth = 128 ) @@ -1562,6 +1562,11 @@ func TestStackMem(t *testing.T) { for i := 0; i < BatchSize; i++ { <-c } + + // The goroutines have signaled via c that they are ready to exit. + // Give them a chance to exit by sleeping. If we don't wait, we + // might not reuse them on the next batch. + time.Sleep(10 * time.Millisecond) } s1 := new(MemStats) ReadMemStats(s1) @@ -1571,7 +1576,9 @@ func TestStackMem(t *testing.T) { if consumed > estimate { t.Fatalf("Stack mem: want %v, got %v", estimate, consumed) } - if s1.StackInuse > 4<<20 { - t.Fatalf("Stack inuse: want %v, got %v", 4<<20, s1.StackInuse) + inuse := s1.StackInuse - s0.StackInuse + t.Logf("Inuse %vMB for stack mem", inuse>>20) + if inuse > 4<<20 { + t.Fatalf("Stack inuse: want %v, got %v", 4<<20, inuse) } } diff --git a/src/pkg/runtime/string_test.go b/src/pkg/runtime/string_test.go index 8f13f0f42..6ba3c1d29 100644 --- a/src/pkg/runtime/string_test.go +++ b/src/pkg/runtime/string_test.go @@ -1,3 +1,7 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package runtime_test import ( diff --git a/src/pkg/runtime/symtab.c b/src/pkg/runtime/symtab.c index d7221c476..578406247 100644 --- a/src/pkg/runtime/symtab.c +++ b/src/pkg/runtime/symtab.c @@ -670,6 +670,6 @@ runtime·showframe(Func *f, bool current) if(current && m->throwing > 0) return 1; if(traceback < 0) - traceback = runtime·gotraceback(); + traceback = runtime·gotraceback(nil); return traceback > 1 || f != nil && contains(f->name, ".") && !hasprefix(f->name, "runtime."); } diff --git a/src/pkg/runtime/sys_darwin_386.s b/src/pkg/runtime/sys_darwin_386.s index 8a938f9f4..59bb9d80d 100644 --- a/src/pkg/runtime/sys_darwin_386.s +++ b/src/pkg/runtime/sys_darwin_386.s @@ -24,18 +24,34 @@ TEXT runtime·exit1(SB),7,$0 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$0 + MOVL $5, AX + INT $0x80 + RET + +TEXT runtime·close(SB),7,$0 + MOVL $6, AX + INT $0x80 + RET + +TEXT runtime·read(SB),7,$0 + MOVL $3, AX + INT $0x80 + RET + TEXT runtime·write(SB),7,$0 MOVL $4, AX INT $0x80 RET -TEXT runtime·raisesigpipe(SB),7,$8 - get_tls(CX) - MOVL m(CX), DX - MOVL m_procid(DX), DX - MOVL DX, 0(SP) // thread_port - MOVL $13, 4(SP) // signal: SIGPIPE - MOVL $328, AX // __pthread_kill +TEXT runtime·raise(SB),7,$16 + MOVL $20, AX // getpid + INT $0x80 + MOVL AX, 4(SP) // pid + MOVL sig+0(FP), AX + MOVL AX, 8(SP) // signal + MOVL $1, 12(SP) // posix + MOVL $37, AX // kill INT $0x80 RET @@ -473,3 +489,32 @@ TEXT runtime·sysctl(SB),7,$0 RET MOVL $0, AX RET + +// int32 runtime·kqueue(void); +TEXT runtime·kqueue(SB),7,$0 + MOVL $362, AX + INT $0x80 + JAE 2(PC) + NEGL AX + RET + +// int32 runtime·kevent(int kq, Kevent *changelist, int nchanges, Kevent *eventlist, int nevents, Timespec *timeout); +TEXT runtime·kevent(SB),7,$0 + MOVL $363, AX + INT $0x80 + JAE 2(PC) + NEGL AX + RET + +// int32 runtime·closeonexec(int32 fd); +TEXT runtime·closeonexec(SB),7,$32 + MOVL $92, AX // fcntl + // 0(SP) is where the caller PC would be; kernel skips it + MOVL fd+0(FP), BX + MOVL BX, 4(SP) // fd + MOVL $2, 8(SP) // F_SETFD + MOVL $1, 12(SP) // FD_CLOEXEC + INT $0x80 + JAE 2(PC) + NEGL AX + RET diff --git a/src/pkg/runtime/sys_darwin_amd64.s b/src/pkg/runtime/sys_darwin_amd64.s index 4e43a76c3..b324a0424 100644 --- a/src/pkg/runtime/sys_darwin_amd64.s +++ b/src/pkg/runtime/sys_darwin_amd64.s @@ -30,6 +30,28 @@ TEXT runtime·exit1(SB),7,$0 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$0 + MOVQ 8(SP), DI // arg 1 pathname + MOVL 16(SP), SI // arg 2 flags + MOVL 20(SP), DX // arg 3 mode + MOVL $(0x2000000+5), AX // syscall entry + SYSCALL + RET + +TEXT runtime·close(SB),7,$0 + MOVL 8(SP), DI // arg 1 fd + MOVL $(0x2000000+6), AX // syscall entry + SYSCALL + RET + +TEXT runtime·read(SB),7,$0 + MOVL 8(SP), DI // arg 1 fd + MOVQ 16(SP), SI // arg 2 buf + MOVL 24(SP), DX // arg 3 count + MOVL $(0x2000000+3), AX // syscall entry + SYSCALL + RET + TEXT runtime·write(SB),7,$0 MOVL 8(SP), DI // arg 1 fd MOVQ 16(SP), SI // arg 2 buf @@ -38,12 +60,13 @@ TEXT runtime·write(SB),7,$0 SYSCALL RET -TEXT runtime·raisesigpipe(SB),7,$24 - get_tls(CX) - MOVQ m(CX), DX - MOVL $13, DI // arg 1 SIGPIPE - MOVQ m_procid(DX), SI // arg 2 thread_port - MOVL $(0x2000000+328), AX // syscall entry __pthread_kill +TEXT runtime·raise(SB),7,$24 + MOVL $(0x2000000+20), AX // getpid + SYSCALL + MOVQ AX, DI // arg 1 - pid + MOVL sig+0(FP), SI // arg 2 - signal + MOVL $1, DX // arg 3 - posix + MOVL $(0x2000000+37), AX // kill SYSCALL RET @@ -267,7 +290,7 @@ TEXT runtime·bsdthread_create(SB),7,$0 MOVQ $(0x2000000+360), AX // bsdthread_create SYSCALL JCC 3(PC) - NEGL AX + NEGQ AX RET MOVL $0, AX RET @@ -320,7 +343,7 @@ TEXT runtime·bsdthread_register(SB),7,$0 MOVQ $(0x2000000+366), AX // bsdthread_register SYSCALL JCC 3(PC) - NEGL AX + NEGQ AX RET MOVL $0, AX RET @@ -413,7 +436,41 @@ TEXT runtime·sysctl(SB),7,$0 MOVL $(0x2000000+202), AX // syscall entry SYSCALL JCC 3(PC) - NEGL AX + NEGQ AX RET MOVL $0, AX RET + +// int32 runtime·kqueue(void); +TEXT runtime·kqueue(SB),7,$0 + MOVQ $0, DI + MOVQ $0, SI + MOVQ $0, DX + MOVL $(0x2000000+362), AX + SYSCALL + JCC 2(PC) + NEGQ AX + RET + +// int32 runtime·kevent(int kq, Kevent *changelist, int nchanges, Kevent *eventlist, int nevents, Timespec *timeout); +TEXT runtime·kevent(SB),7,$0 + MOVL 8(SP), DI + MOVQ 16(SP), SI + MOVL 24(SP), DX + MOVQ 32(SP), R10 + MOVL 40(SP), R8 + MOVQ 48(SP), R9 + MOVL $(0x2000000+363), AX + SYSCALL + JCC 2(PC) + NEGQ AX + RET + +// void runtime·closeonexec(int32 fd); +TEXT runtime·closeonexec(SB),7,$0 + MOVL 8(SP), DI // fd + MOVQ $2, SI // F_SETFD + MOVQ $1, DX // FD_CLOEXEC + MOVL $(0x2000000+92), AX // fcntl + SYSCALL + RET diff --git a/src/pkg/runtime/sys_freebsd_386.s b/src/pkg/runtime/sys_freebsd_386.s index d5370267a..d960663cb 100644 --- a/src/pkg/runtime/sys_freebsd_386.s +++ b/src/pkg/runtime/sys_freebsd_386.s @@ -56,6 +56,21 @@ TEXT runtime·exit1(SB),7,$-4 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$-4 + MOVL $5, AX + INT $0x80 + RET + +TEXT runtime·close(SB),7,$-4 + MOVL $6, AX + INT $0x80 + RET + +TEXT runtime·read(SB),7,$-4 + MOVL $3, AX + INT $0x80 + RET + TEXT runtime·write(SB),7,$-4 MOVL $4, AX INT $0x80 @@ -66,16 +81,17 @@ TEXT runtime·getrlimit(SB),7,$-4 INT $0x80 RET -TEXT runtime·raisesigpipe(SB),7,$12 +TEXT runtime·raise(SB),7,$16 // thr_self(&8(SP)) LEAL 8(SP), AX - MOVL AX, 0(SP) + MOVL AX, 4(SP) MOVL $432, AX INT $0x80 // thr_kill(self, SIGPIPE) MOVL 8(SP), AX - MOVL AX, 0(SP) - MOVL $13, 4(SP) + MOVL AX, 4(SP) + MOVL sig+0(FP), AX + MOVL AX, 8(SP) MOVL $433, AX INT $0x80 RET diff --git a/src/pkg/runtime/sys_freebsd_amd64.s b/src/pkg/runtime/sys_freebsd_amd64.s index 40c6237e2..cfa33d4fb 100644 --- a/src/pkg/runtime/sys_freebsd_amd64.s +++ b/src/pkg/runtime/sys_freebsd_amd64.s @@ -58,6 +58,28 @@ TEXT runtime·exit1(SB),7,$-8 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$-8 + MOVQ 8(SP), DI // arg 1 pathname + MOVL 16(SP), SI // arg 2 flags + MOVL 20(SP), DX // arg 3 mode + MOVL $5, AX + SYSCALL + RET + +TEXT runtime·close(SB),7,$-8 + MOVL 8(SP), DI // arg 1 fd + MOVL $6, AX + SYSCALL + RET + +TEXT runtime·read(SB),7,$-8 + MOVL 8(SP), DI // arg 1 fd + MOVQ 16(SP), SI // arg 2 buf + MOVL 24(SP), DX // arg 3 count + MOVL $3, AX + SYSCALL + RET + TEXT runtime·write(SB),7,$-8 MOVL 8(SP), DI // arg 1 fd MOVQ 16(SP), SI // arg 2 buf @@ -73,14 +95,14 @@ TEXT runtime·getrlimit(SB),7,$-8 SYSCALL RET -TEXT runtime·raisesigpipe(SB),7,$16 +TEXT runtime·raise(SB),7,$16 // thr_self(&8(SP)) LEAQ 8(SP), DI // arg 1 &8(SP) MOVL $432, AX SYSCALL // thr_kill(self, SIGPIPE) MOVQ 8(SP), DI // arg 1 id - MOVQ $13, SI // arg 2 SIGPIPE + MOVL sig+0(FP), SI // arg 2 MOVL $433, AX SYSCALL RET @@ -239,7 +261,7 @@ TEXT runtime·sysctl(SB),7,$0 MOVQ $202, AX // sys___sysctl SYSCALL JCC 3(PC) - NEGL AX + NEGQ AX RET MOVL $0, AX RET diff --git a/src/pkg/runtime/sys_freebsd_arm.s b/src/pkg/runtime/sys_freebsd_arm.s index 77050e8d0..5531936ff 100644 --- a/src/pkg/runtime/sys_freebsd_arm.s +++ b/src/pkg/runtime/sys_freebsd_arm.s @@ -54,6 +54,20 @@ TEXT runtime·exit1(SB),7,$-8 MOVW.CS R9, (R9) RET +TEXT runtime·open(SB),7,$-8 + MOVW 0(FP), R0 // arg 1 name + MOVW 4(FP), R1 // arg 2 mode + MOVW 8(FP), R2 // arg 3 perm + SWI $5 + RET + +TEXT runtime·read(SB),7,$-8 + MOVW 0(FP), R0 // arg 1 fd + MOVW 4(FP), R1 // arg 2 buf + MOVW 8(FP), R2 // arg 3 count + SWI $3 + RET + TEXT runtime·write(SB),7,$-8 MOVW 0(FP), R0 // arg 1 fd MOVW 4(FP), R1 // arg 2 buf @@ -61,6 +75,11 @@ TEXT runtime·write(SB),7,$-8 SWI $4 RET +TEXT runtime·close(SB),7,$-8 + MOVW 0(FP), R0 // arg 1 fd + SWI $6 + RET + TEXT runtime·getrlimit(SB),7,$-8 MOVW 0(FP), R0 MOVW 4(FP), R1 @@ -68,13 +87,13 @@ TEXT runtime·getrlimit(SB),7,$-8 SWI $194 RET -TEXT runtime·raisesigpipe(SB),7,$8 +TEXT runtime·raise(SB),7,$8 // thr_self(&4(R13)) MOVW $4(R13), R0 // arg 1 &4(R13) SWI $432 // thr_kill(self, SIGPIPE) MOVW 4(R13), R0 // arg 1 id - MOVW $13, R1 // arg 2 SIGPIPE + MOVW sig+0(FP), R1 // arg 2 - signal SWI $433 RET diff --git a/src/pkg/runtime/sys_linux_386.s b/src/pkg/runtime/sys_linux_386.s index f27fd4713..76ebe3dcf 100644 --- a/src/pkg/runtime/sys_linux_386.s +++ b/src/pkg/runtime/sys_linux_386.s @@ -77,11 +77,11 @@ TEXT runtime·usleep(SB),7,$8 CALL *runtime·_vdso(SB) RET -TEXT runtime·raisesigpipe(SB),7,$12 +TEXT runtime·raise(SB),7,$12 MOVL $224, AX // syscall - gettid CALL *runtime·_vdso(SB) - MOVL AX, 0(SP) // arg 1 tid - MOVL $13, 4(SP) // arg 2 SIGPIPE + MOVL AX, BX // arg 1 tid + MOVL sig+0(FP), CX // arg 2 signal MOVL $238, AX // syscall - tkill CALL *runtime·_vdso(SB) RET @@ -430,3 +430,46 @@ TEXT runtime·sched_getaffinity(SB),7,$0 MOVL 12(SP), DX CALL *runtime·_vdso(SB) RET + +// int32 runtime·epollcreate(int32 size); +TEXT runtime·epollcreate(SB),7,$0 + MOVL $254, AX + MOVL 4(SP), BX + CALL *runtime·_vdso(SB) + RET + +// int32 runtime·epollcreate1(int32 flags); +TEXT runtime·epollcreate1(SB),7,$0 + MOVL $329, AX + MOVL 4(SP), BX + CALL *runtime·_vdso(SB) + RET + +// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); +TEXT runtime·epollctl(SB),7,$0 + MOVL $255, AX + MOVL 4(SP), BX + MOVL 8(SP), CX + MOVL 12(SP), DX + MOVL 16(SP), SI + CALL *runtime·_vdso(SB) + RET + +// int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); +TEXT runtime·epollwait(SB),7,$0 + MOVL $256, AX + MOVL 4(SP), BX + MOVL 8(SP), CX + MOVL 12(SP), DX + MOVL 16(SP), SI + CALL *runtime·_vdso(SB) + RET + +// void runtime·closeonexec(int32 fd); +TEXT runtime·closeonexec(SB),7,$0 + MOVL $55, AX // fcntl + MOVL 4(SP), BX // fd + MOVL $2, CX // F_SETFD + MOVL $1, DX // FD_CLOEXEC + CALL *runtime·_vdso(SB) + RET diff --git a/src/pkg/runtime/sys_linux_amd64.s b/src/pkg/runtime/sys_linux_amd64.s index e45943758..2d802abb6 100644 --- a/src/pkg/runtime/sys_linux_amd64.s +++ b/src/pkg/runtime/sys_linux_amd64.s @@ -75,11 +75,11 @@ TEXT runtime·usleep(SB),7,$16 SYSCALL RET -TEXT runtime·raisesigpipe(SB),7,$12 +TEXT runtime·raise(SB),7,$12 MOVL $186, AX // syscall - gettid SYSCALL MOVL AX, DI // arg 1 tid - MOVL $13, SI // arg 2 SIGPIPE + MOVL sig+0(FP), SI // arg 2 MOVL $200, AX // syscall - tkill SYSCALL RET @@ -347,3 +347,46 @@ TEXT runtime·sched_getaffinity(SB),7,$0 MOVL $204, AX // syscall entry SYSCALL RET + +// int32 runtime·epollcreate(int32 size); +TEXT runtime·epollcreate(SB),7,$0 + MOVL 8(SP), DI + MOVL $213, AX // syscall entry + SYSCALL + RET + +// int32 runtime·epollcreate1(int32 flags); +TEXT runtime·epollcreate1(SB),7,$0 + MOVL 8(SP), DI + MOVL $291, AX // syscall entry + SYSCALL + RET + +// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev); +TEXT runtime·epollctl(SB),7,$0 + MOVL 8(SP), DI + MOVL 12(SP), SI + MOVL 16(SP), DX + MOVQ 24(SP), R10 + MOVL $233, AX // syscall entry + SYSCALL + RET + +// int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout); +TEXT runtime·epollwait(SB),7,$0 + MOVL 8(SP), DI + MOVQ 16(SP), SI + MOVL 24(SP), DX + MOVL 28(SP), R10 + MOVL $232, AX // syscall entry + SYSCALL + RET + +// void runtime·closeonexec(int32 fd); +TEXT runtime·closeonexec(SB),7,$0 + MOVL 8(SP), DI // fd + MOVQ $2, SI // F_SETFD + MOVQ $1, DX // FD_CLOEXEC + MOVL $72, AX // fcntl + SYSCALL + RET diff --git a/src/pkg/runtime/sys_linux_arm.s b/src/pkg/runtime/sys_linux_arm.s index 8bae2933f..7f813482d 100644 --- a/src/pkg/runtime/sys_linux_arm.s +++ b/src/pkg/runtime/sys_linux_arm.s @@ -36,6 +36,11 @@ #define SYS_ugetrlimit (SYS_BASE + 191) #define SYS_sched_getaffinity (SYS_BASE + 242) #define SYS_clock_gettime (SYS_BASE + 263) +#define SYS_epoll_create (SYS_BASE + 250) +#define SYS_epoll_ctl (SYS_BASE + 251) +#define SYS_epoll_wait (SYS_BASE + 252) +#define SYS_epoll_create1 (SYS_BASE + 357) +#define SYS_fcntl (SYS_BASE + 55) #define ARM_BASE (SYS_BASE + 0x0f0000) @@ -92,11 +97,11 @@ TEXT runtime·exit1(SB),7,$-4 MOVW $1003, R1 MOVW R0, (R1) // fail hard -TEXT runtime·raisesigpipe(SB),7,$-4 +TEXT runtime·raise(SB),7,$-4 MOVW $SYS_gettid, R7 SWI $0 // arg 1 tid already in R0 from gettid - MOVW $13, R1 // arg 2 SIGPIPE + MOVW sig+0(FP), R1 // arg 2 - signal MOVW $SYS_tkill, R7 SWI $0 RET @@ -371,7 +376,6 @@ cascheck: MOVW $0, R0 RET - TEXT runtime·casp(SB),7,$0 B runtime·cas(SB) @@ -387,3 +391,46 @@ TEXT runtime·sched_getaffinity(SB),7,$0 MOVW $SYS_sched_getaffinity, R7 SWI $0 RET + +// int32 runtime·epollcreate(int32 size) +TEXT runtime·epollcreate(SB),7,$0 + MOVW 0(FP), R0 + MOVW $SYS_epoll_create, R7 + SWI $0 + RET + +// int32 runtime·epollcreate1(int32 flags) +TEXT runtime·epollcreate1(SB),7,$0 + MOVW 0(FP), R0 + MOVW $SYS_epoll_create1, R7 + SWI $0 + RET + +// int32 runtime·epollctl(int32 epfd, int32 op, int32 fd, EpollEvent *ev) +TEXT runtime·epollctl(SB),7,$0 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + MOVW 12(FP), R3 + MOVW $SYS_epoll_ctl, R7 + SWI $0 + RET + +// int32 runtime·epollwait(int32 epfd, EpollEvent *ev, int32 nev, int32 timeout) +TEXT runtime·epollwait(SB),7,$0 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + MOVW 12(FP), R3 + MOVW $SYS_epoll_wait, R7 + SWI $0 + RET + +// void runtime·closeonexec(int32 fd) +TEXT runtime·closeonexec(SB),7,$0 + MOVW 0(FP), R0 // fd + MOVW $2, R1 // F_SETFD + MOVW $1, R2 // FD_CLOEXEC + MOVW $SYS_fcntl, R7 + SWI $0 + RET diff --git a/src/pkg/runtime/sys_netbsd_386.s b/src/pkg/runtime/sys_netbsd_386.s index 3d3d31273..992eba77d 100644 --- a/src/pkg/runtime/sys_netbsd_386.s +++ b/src/pkg/runtime/sys_netbsd_386.s @@ -22,6 +22,21 @@ TEXT runtime·exit1(SB),7,$-4 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$-4 + MOVL $5, AX + INT $0x80 + RET + +TEXT runtime·close(SB),7,$-4 + MOVL $6, AX + INT $0x80 + RET + +TEXT runtime·read(SB),7,$-4 + MOVL $3, AX + INT $0x80 + RET + TEXT runtime·write(SB),7,$-4 MOVL $4, AX // sys_write INT $0x80 @@ -46,12 +61,13 @@ TEXT runtime·usleep(SB),7,$24 INT $0x80 RET -TEXT runtime·raisesigpipe(SB),7,$12 +TEXT runtime·raise(SB),7,$12 MOVL $311, AX // sys__lwp_self INT $0x80 MOVL $0, 0(SP) MOVL AX, 4(SP) // arg 1 - target - MOVL $13, 8(SP) // arg 2 - signo == SIGPIPE + MOVL sig+0(FP), AX + MOVL AX, 8(SP) // arg 2 - signo MOVL $318, AX // sys__lwp_kill INT $0x80 RET @@ -72,8 +88,6 @@ TEXT runtime·mmap(SB),7,$36 STOSL MOVL $197, AX // sys_mmap INT $0x80 - JCC 2(PC) - NEGL AX RET TEXT runtime·munmap(SB),7,$-4 diff --git a/src/pkg/runtime/sys_netbsd_amd64.s b/src/pkg/runtime/sys_netbsd_amd64.s index e73e83ded..574d8a91b 100644 --- a/src/pkg/runtime/sys_netbsd_amd64.s +++ b/src/pkg/runtime/sys_netbsd_amd64.s @@ -16,7 +16,7 @@ TEXT runtime·lwp_create(SB),7,$0 MOVL $309, AX // sys__lwp_create SYSCALL JCC 2(PC) - NEGL AX + NEGQ AX RET TEXT runtime·lwp_tramp(SB),7,$0 @@ -79,6 +79,28 @@ TEXT runtime·exit1(SB),7,$-8 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$-8 + MOVQ 8(SP), DI // arg 1 pathname + MOVL 16(SP), SI // arg 2 flags + MOVL 20(SP), DX // arg 3 mode + MOVL $5, AX + SYSCALL + RET + +TEXT runtime·close(SB),7,$-8 + MOVL 8(SP), DI // arg 1 fd + MOVL $6, AX + SYSCALL + RET + +TEXT runtime·read(SB),7,$-8 + MOVL 8(SP), DI // arg 1 fd + MOVQ 16(SP), SI // arg 2 buf + MOVL 24(SP), DX // arg 3 count + MOVL $3, AX + SYSCALL + RET + TEXT runtime·write(SB),7,$-8 MOVL 8(SP), DI // arg 1 - fd MOVQ 16(SP), SI // arg 2 - buf @@ -103,11 +125,11 @@ TEXT runtime·usleep(SB),7,$16 SYSCALL RET -TEXT runtime·raisesigpipe(SB),7,$16 +TEXT runtime·raise(SB),7,$16 MOVL $311, AX // sys__lwp_self SYSCALL MOVQ AX, DI // arg 1 - target - MOVQ $13, SI // arg 2 - signo == SIGPIPE + MOVL sig+0(FP), SI // arg 2 - signo MOVL $318, AX // sys__lwp_kill SYSCALL RET @@ -231,8 +253,6 @@ TEXT runtime·mmap(SB),7,$0 MOVQ $0, R9 // arg 6 - pad MOVL $197, AX // sys_mmap SYSCALL - JCC 2(PC) - NEGL AX ADDQ $16, SP RET @@ -284,7 +304,7 @@ TEXT runtime·sysctl(SB),7,$0 MOVQ $202, AX // sys___sysctl SYSCALL JCC 3(PC) - NEGL AX + NEGQ AX RET MOVL $0, AX RET diff --git a/src/pkg/runtime/sys_netbsd_arm.s b/src/pkg/runtime/sys_netbsd_arm.s index 4a119c5de..3ff335f4d 100644 --- a/src/pkg/runtime/sys_netbsd_arm.s +++ b/src/pkg/runtime/sys_netbsd_arm.s @@ -21,6 +21,25 @@ TEXT runtime·exit1(SB),7,$-4 MOVW $1, R9 // crash MOVW R9, (R9) RET + +TEXT runtime·open(SB),7,$-8 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + SWI $0xa00005 + RET + +TEXT runtime·close(SB),7,$-8 + MOVW 0(FP), R0 + SWI $0xa00006 + RET + +TEXT runtime·read(SB),7,$-8 + MOVW 0(FP), R0 + MOVW 4(FP), R1 + MOVW 8(FP), R2 + SWI $0xa00003 + RET TEXT runtime·write(SB),7,$-4 MOVW 0(FP), R0 // arg 1 - fd @@ -88,9 +107,9 @@ TEXT runtime·usleep(SB),7,$16 SWI $0xa001ae // sys_nanosleep RET -TEXT runtime·raisesigpipe(SB),7,$16 +TEXT runtime·raise(SB),7,$16 SWI $0xa00137 // sys__lwp_self, the returned R0 is arg 1 - MOVW $13, R1 // arg 2 - signo == SIGPIPE + MOVW sig+0(FP), R1 // arg 2 - signal SWI $0xa0013e // sys__lwp_kill RET diff --git a/src/pkg/runtime/sys_openbsd_386.s b/src/pkg/runtime/sys_openbsd_386.s index c62e0f949..37b6ff215 100644 --- a/src/pkg/runtime/sys_openbsd_386.s +++ b/src/pkg/runtime/sys_openbsd_386.s @@ -24,6 +24,21 @@ TEXT runtime·exit1(SB),7,$8 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$-4 + MOVL $5, AX + INT $0x80 + RET + +TEXT runtime·close(SB),7,$-4 + MOVL $6, AX + INT $0x80 + RET + +TEXT runtime·read(SB),7,$-4 + MOVL $3, AX + INT $0x80 + RET + TEXT runtime·write(SB),7,$-4 MOVL $4, AX // sys_write INT $0x80 @@ -47,12 +62,13 @@ TEXT runtime·usleep(SB),7,$20 INT $0x80 RET -TEXT runtime·raisesigpipe(SB),7,$12 +TEXT runtime·raise(SB),7,$12 MOVL $299, AX // sys_getthrid INT $0x80 MOVL $0, 0(SP) MOVL AX, 4(SP) // arg 1 - pid - MOVL $13, 8(SP) // arg 2 - signum == SIGPIPE + MOVL sig+0(FP), AX + MOVL AX, 8(SP) // arg 2 - signum MOVL $37, AX // sys_kill INT $0x80 RET @@ -73,8 +89,6 @@ TEXT runtime·mmap(SB),7,$36 STOSL MOVL $197, AX // sys_mmap INT $0x80 - JCC 2(PC) - NEGL AX RET TEXT runtime·munmap(SB),7,$-4 diff --git a/src/pkg/runtime/sys_openbsd_amd64.s b/src/pkg/runtime/sys_openbsd_amd64.s index 8a736507f..cbd2c2f76 100644 --- a/src/pkg/runtime/sys_openbsd_amd64.s +++ b/src/pkg/runtime/sys_openbsd_amd64.s @@ -23,7 +23,7 @@ TEXT runtime·tfork(SB),7,$32 // Return if tfork syscall failed. JCC 3(PC) - NEGL AX + NEGQ AX RET // In parent, return. @@ -87,6 +87,28 @@ TEXT runtime·exit1(SB),7,$-8 MOVL $0xf1, 0xf1 // crash RET +TEXT runtime·open(SB),7,$-8 + MOVQ 8(SP), DI // arg 1 pathname + MOVL 16(SP), SI // arg 2 flags + MOVL 20(SP), DX // arg 3 mode + MOVL $5, AX + SYSCALL + RET + +TEXT runtime·close(SB),7,$-8 + MOVL 8(SP), DI // arg 1 fd + MOVL $6, AX + SYSCALL + RET + +TEXT runtime·read(SB),7,$-8 + MOVL 8(SP), DI // arg 1 fd + MOVQ 16(SP), SI // arg 2 buf + MOVL 24(SP), DX // arg 3 count + MOVL $3, AX + SYSCALL + RET + TEXT runtime·write(SB),7,$-8 MOVL 8(SP), DI // arg 1 - fd MOVQ 16(SP), SI // arg 2 - buf @@ -111,11 +133,11 @@ TEXT runtime·usleep(SB),7,$16 SYSCALL RET -TEXT runtime·raisesigpipe(SB),7,$16 +TEXT runtime·raise(SB),7,$16 MOVL $299, AX // sys_getthrid SYSCALL MOVQ AX, DI // arg 1 - pid - MOVQ $13, SI // arg 2 - signum == SIGPIPE + MOVL sig+0(FP), SI // arg 2 - signum MOVL $37, AX // sys_kill SYSCALL RET @@ -220,8 +242,6 @@ TEXT runtime·mmap(SB),7,$0 MOVQ $0, R9 // arg 6 - pad MOVL $197, AX SYSCALL - JCC 2(PC) - NEGL AX ADDQ $16, SP RET @@ -272,7 +292,7 @@ TEXT runtime·sysctl(SB),7,$0 MOVQ $202, AX // sys___sysctl SYSCALL JCC 3(PC) - NEGL AX + NEGQ AX RET MOVL $0, AX RET diff --git a/src/pkg/runtime/sys_plan9_386.s b/src/pkg/runtime/sys_plan9_386.s index 3385b083a..1f860a961 100644 --- a/src/pkg/runtime/sys_plan9_386.s +++ b/src/pkg/runtime/sys_plan9_386.s @@ -170,3 +170,30 @@ TEXT runtime·sigtramp(SB),7,$0 // Only used by the 64-bit runtime. TEXT runtime·setfpmasks(SB),7,$0 RET + +#define ERRMAX 128 /* from os_plan9.h */ + +// func errstr() String +// Only used by package syscall. +// Grab error string due to a syscall made +// in entersyscall mode, without going +// through the allocator (issue 4994). +// See ../syscall/asm_plan9_386.s:/·Syscall/ +TEXT runtime·errstr(SB),7,$0 + get_tls(AX) + MOVL m(AX), BX + MOVL m_errstr(BX), CX + MOVL CX, 4(SP) + MOVL $ERRMAX, 8(SP) + MOVL $41, AX + INT $64 + + // syscall requires caller-save + MOVL 4(SP), CX + + // push the argument + PUSHL CX + CALL runtime·findnull(SB) + POPL CX + MOVL AX, 8(SP) + RET diff --git a/src/pkg/runtime/sys_plan9_amd64.s b/src/pkg/runtime/sys_plan9_amd64.s index b34f98a68..c0c896ebc 100644 --- a/src/pkg/runtime/sys_plan9_amd64.s +++ b/src/pkg/runtime/sys_plan9_amd64.s @@ -206,3 +206,31 @@ TEXT runtime·setfpmasks(SB),7,$8 MOVL AX, 0(SP) LDMXCSR 0(SP) RET + +#define ERRMAX 128 /* from os_plan9.h */ + +// func errstr() String +// Only used by package syscall. +// Grab error string due to a syscall made +// in entersyscall mode, without going +// through the allocator (issue 4994). +// See ../syscall/asm_plan9_386.s:/·Syscall/ +TEXT runtime·errstr(SB),7,$0 + get_tls(AX) + MOVQ m(AX), BX + MOVQ m_errstr(BX), CX + MOVQ CX, 8(SP) + MOVQ $ERRMAX, 16(SP) + MOVQ $0x8000, AX + MOVQ $41, BP + SYSCALL + + // syscall requires caller-save + MOVQ 8(SP), CX + + // push the argument + PUSHQ CX + CALL runtime·findnull(SB) + POPQ CX + MOVQ AX, 16(SP) + RET diff --git a/src/pkg/runtime/sys_windows_386.s b/src/pkg/runtime/sys_windows_386.s index ca59f0a1d..206cdccc4 100644 --- a/src/pkg/runtime/sys_windows_386.s +++ b/src/pkg/runtime/sys_windows_386.s @@ -314,3 +314,46 @@ TEXT runtime·remove_exception_handler(SB),7,$0 MOVL AX, 0(FS) RET + +TEXT runtime·osyield(SB),7,$20 + // Tried NtYieldExecution but it doesn't yield hard enough. + // NtWaitForSingleObject being used here as Sleep(0). + MOVL runtime·NtWaitForSingleObject(SB), AX + MOVL $-1, hi-4(SP) + MOVL $-1, lo-8(SP) + LEAL lo-8(SP), BX + MOVL BX, ptime-12(SP) + MOVL $0, alertable-16(SP) + MOVL $-1, handle-20(SP) + MOVL SP, BP + CALL checkstack4<>(SB) + CALL AX + MOVL BP, SP + RET + +TEXT runtime·usleep(SB),7,$20 + MOVL runtime·NtWaitForSingleObject(SB), AX + // Have 1us units; need negative 100ns units. + // Assume multiply by 10 will not overflow 32-bit word. + MOVL usec+0(FP), BX + IMULL $10, BX + NEGL BX + MOVL $-1, hi-4(SP) + MOVL BX, lo-8(SP) + LEAL lo-8(SP), BX + MOVL BX, ptime-12(SP) + MOVL $0, alertable-16(SP) + MOVL $-1, handle-20(SP) + MOVL SP, BP + CALL checkstack4<>(SB) + CALL AX + MOVL BP, SP + RET + +// This function requires 4 bytes of stack, +// to simulate what calling NtWaitForSingleObject will use. +// (It is just a CALL to the system call dispatch.) +// If the linker okays the call to checkstack4 (a NOSPLIT function) +// then the call to NtWaitForSingleObject is okay too. +TEXT checkstack4<>(SB),7,$4 + RET diff --git a/src/pkg/runtime/sys_windows_amd64.s b/src/pkg/runtime/sys_windows_amd64.s index fe88f3b75..c20a268b1 100644 --- a/src/pkg/runtime/sys_windows_amd64.s +++ b/src/pkg/runtime/sys_windows_amd64.s @@ -346,3 +346,35 @@ TEXT runtime·install_exception_handler(SB),7,$0 TEXT runtime·remove_exception_handler(SB),7,$0 RET + +TEXT runtime·osyield(SB),7,$8 + // Tried NtYieldExecution but it doesn't yield hard enough. + // NtWaitForSingleObject being used here as Sleep(0). + // The CALL is safe because NtXxx is a system call wrapper: + // it puts the right system call number in AX, then does + // a SYSENTER and a RET. + MOVQ runtime·NtWaitForSingleObject(SB), AX + MOVQ $1, BX + NEGQ BX + MOVQ SP, R8 // ptime + MOVQ BX, (R8) + MOVQ $-1, CX // handle + MOVQ $0, DX // alertable + CALL AX + RET + +TEXT runtime·usleep(SB),7,$8 + // The CALL is safe because NtXxx is a system call wrapper: + // it puts the right system call number in AX, then does + // a SYSENTER and a RET. + MOVQ runtime·NtWaitForSingleObject(SB), AX + // Have 1us units; want negative 100ns units. + MOVL usec+0(FP), BX + IMULQ $10, BX + NEGQ BX + MOVQ SP, R8 // ptime + MOVQ BX, (R8) + MOVQ $-1, CX // handle + MOVQ $0, DX // alertable + CALL AX + RET diff --git a/src/pkg/runtime/time.goc b/src/pkg/runtime/time.goc index 2babb173d..6de989f51 100644 --- a/src/pkg/runtime/time.goc +++ b/src/pkg/runtime/time.goc @@ -15,7 +15,6 @@ package time static Timers timers; static void addtimer(Timer*); -static bool deltimer(Timer*); // Package time APIs. // Godoc uses the comments in package time, not these. @@ -31,15 +30,13 @@ func Sleep(ns int64) { func startTimer(t *Timer) { if(raceenabled) runtime·racerelease(t); - runtime·lock(&timers); - addtimer(t); - runtime·unlock(&timers); + runtime·addtimer(t); } // stopTimer removes t from the timer heap if it is there. // It returns true if t was removed, false if t wasn't even there. func stopTimer(t *Timer) (stopped bool) { - stopped = deltimer(t); + stopped = runtime·deltimer(t); } // C runtime. @@ -79,6 +76,14 @@ runtime·tsleep(int64 ns, int8 *reason) static FuncVal timerprocv = {timerproc}; +void +runtime·addtimer(Timer *t) +{ + runtime·lock(&timers); + addtimer(t); + runtime·unlock(&timers); +} + // Add a timer to the heap and start or kick the timer proc // if the new timer is earlier than any of the others. static void @@ -121,8 +126,8 @@ addtimer(Timer *t) // Delete timer t from the heap. // Do not need to update the timerproc: // if it wakes up early, no big deal. -static bool -deltimer(Timer *t) +bool +runtime·deltimer(Timer *t) { int32 i; diff --git a/src/pkg/runtime/time_plan9_386.c b/src/pkg/runtime/time_plan9_386.c index a29d45715..fc08a90d6 100644 --- a/src/pkg/runtime/time_plan9_386.c +++ b/src/pkg/runtime/time_plan9_386.c @@ -24,7 +24,7 @@ runtime·nanotime(void) // file descriptor) is roughly four times slower // in 9vx on a 2.16 GHz Intel Core 2 Duo. - if(fd < 0 && (fd = runtime·open((byte*)"/dev/bintime", OREAD|OCEXEC)) < 0) + if(fd < 0 && (fd = runtime·open("/dev/bintime", OREAD|OCEXEC, 0)) < 0) return 0; if(runtime·pread(fd, b, sizeof b, 0) != sizeof b) return 0; diff --git a/src/pkg/runtime/vdso_linux_amd64.c b/src/pkg/runtime/vdso_linux_amd64.c index ab68c23c3..f55d312a0 100644 --- a/src/pkg/runtime/vdso_linux_amd64.c +++ b/src/pkg/runtime/vdso_linux_amd64.c @@ -4,6 +4,7 @@ #include "runtime.h" +#define AT_RANDOM 25 #define AT_SYSINFO_EHDR 33 #define AT_NULL 0 /* End of vector */ #define PT_LOAD 1 /* Loadable program segment */ @@ -319,11 +320,16 @@ runtime·linux_setup_vdso(int32 argc, uint8** argv) if(elf_auxv[i].a_type == AT_SYSINFO_EHDR) { if(elf_auxv[i].a_un.a_val == 0) { // Something went wrong - return; + continue; } vdso_init_from_sysinfo_ehdr(&vdso_info, (Elf64_Ehdr*)elf_auxv[i].a_un.a_val); vdso_parse_symbols(&vdso_info, vdso_find_version(&vdso_info, &linux26)); - return; + continue; + } + if(elf_auxv[i].a_type == AT_RANDOM) { + runtime·startup_random_data = (byte*)elf_auxv[i].a_un.a_val; + runtime·startup_random_data_len = 16; + continue; } } } |