diff options
Diffstat (limited to 'src/pkg/runtime')
-rw-r--r-- | src/pkg/runtime/alg.c | 28 | ||||
-rw-r--r-- | src/pkg/runtime/extern.go | 5 | ||||
-rw-r--r-- | src/pkg/runtime/hashmap.c | 161 | ||||
-rw-r--r-- | src/pkg/runtime/malloc.goc | 1 | ||||
-rw-r--r-- | src/pkg/runtime/mfinal.c | 3 | ||||
-rw-r--r-- | src/pkg/runtime/mgc0.c | 1 | ||||
-rw-r--r-- | src/pkg/runtime/softfloat_arm.c | 65 | ||||
-rw-r--r-- | src/pkg/runtime/time.goc | 5 | ||||
-rw-r--r-- | src/pkg/runtime/vlrt_arm.c | 2 |
9 files changed, 192 insertions, 79 deletions
diff --git a/src/pkg/runtime/alg.c b/src/pkg/runtime/alg.c index e3c42916e..36973eba3 100644 --- a/src/pkg/runtime/alg.c +++ b/src/pkg/runtime/alg.c @@ -5,6 +5,9 @@ #include "runtime.h" #include "type.h" +#define M0 (sizeof(uintptr)==4 ? 2860486313UL : 33054211828000289ULL) +#define M1 (sizeof(uintptr)==4 ? 3267000013UL : 23344194077549503ULL) + /* * map and chan helpers for * dealing with unknown types @@ -16,19 +19,13 @@ runtime·memhash(uintptr *h, uintptr s, void *a) uintptr hash; b = a; - if(sizeof(hash) == 4) - hash = 2860486313U; - else - hash = 33054211828000289ULL; + hash = M0; while(s > 0) { - if(sizeof(hash) == 4) - hash = (hash ^ *b) * 3267000013UL; - else - hash = (hash ^ *b) * 23344194077549503ULL; + hash = (hash ^ *b) * M1; b++; s--; } - *h ^= hash; + *h = (*h ^ hash) * M1; } void @@ -252,7 +249,7 @@ runtime·f32hash(uintptr *h, uintptr s, void *a) hash = runtime·fastrand1(); // any kind of NaN else hash = *(uint32*)a; - *h ^= (*h ^ hash ^ 2860486313U) * 3267000013U; + *h = (*h ^ hash ^ M0) * M1; } void @@ -271,14 +268,11 @@ runtime·f64hash(uintptr *h, uintptr s, void *a) else { u = *(uint64*)a; if(sizeof(uintptr) == 4) - hash = ((uint32)(u>>32) * 3267000013UL) ^ (uint32)u; + hash = ((uint32)(u>>32) * M1) ^ (uint32)u; else hash = u; } - if(sizeof(uintptr) == 4) - *h = (*h ^ hash ^ 2860486313U) * 3267000013U; - else - *h = (*h ^ hash ^ 33054211828000289ULL) * 23344194077549503ULL; + *h = (*h ^ hash ^ M0) * M1; } void @@ -357,7 +351,7 @@ void runtime·interhash(uintptr *h, uintptr s, void *a) { USED(s); - *h ^= runtime·ifacehash(*(Iface*)a); + *h = (*h ^ runtime·ifacehash(*(Iface*)a)) * M1; } void @@ -391,7 +385,7 @@ void runtime·nilinterhash(uintptr *h, uintptr s, void *a) { USED(s); - *h ^= runtime·efacehash(*(Eface*)a); + *h = (*h ^ runtime·efacehash(*(Eface*)a)) * M1; } void diff --git a/src/pkg/runtime/extern.go b/src/pkg/runtime/extern.go index f9c5b8e3d..d93259d7b 100644 --- a/src/pkg/runtime/extern.go +++ b/src/pkg/runtime/extern.go @@ -20,7 +20,7 @@ func Goexit() // Caller reports file and line number information about function invocations on // the calling goroutine's stack. The argument skip is the number of stack frames -// to ascend, with 1 identifying the caller of Caller. (For historical reasons the +// to ascend, with 0 identifying the caller of Caller. (For historical reasons the // meaning of skip differs between Caller and Callers.) The return values report the // program counter, file name, and line number within the file of the corresponding // call. The boolean ok is false if it was not possible to recover the information. @@ -28,7 +28,8 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) // Callers fills the slice pc with the program counters of function invocations // on the calling goroutine's stack. The argument skip is the number of stack frames -// to skip before recording in pc, with 0 starting at the caller of Callers. +// to skip before recording in pc, with 0 identifying the frame for Callers itself and +// 1 identifying the caller of Callers. // It returns the number of entries written to pc. func Callers(skip int, pc []uintptr) int diff --git a/src/pkg/runtime/hashmap.c b/src/pkg/runtime/hashmap.c index 1def96727..63ed4e2a3 100644 --- a/src/pkg/runtime/hashmap.c +++ b/src/pkg/runtime/hashmap.c @@ -6,17 +6,24 @@ #include "hashmap.h" #include "type.h" +/* Hmap flag values */ +#define IndirectVal (1<<0) /* storing pointers to values */ +#define IndirectKey (1<<1) /* storing pointers to keys */ +#define CanFreeTable (1<<2) /* okay to free subtables */ +#define CanFreeKey (1<<3) /* okay to free pointers to keys */ + struct Hmap { /* a hash table; initialize with hash_init() */ uint32 count; /* elements in table - must be first */ uint8 datasize; /* amount of data to store in entry */ - uint8 max_power; /* max power of 2 to create sub-tables */ - uint8 indirectval; /* storing pointers to values */ + uint8 flag; uint8 valoff; /* offset of value in key+value data block */ int32 changes; /* inc'ed whenever a subtable is created/grown */ uintptr hash0; /* hash seed */ struct hash_subtable *st; /* first-level table */ }; +#define MaxData 255 + struct hash_entry { hash_hash_t hash; /* hash value of data */ byte data[1]; /* user data has "datasize" bytes */ @@ -54,6 +61,7 @@ struct hash_subtable { ((struct hash_entry *) (((byte *) (base)) + (byte_offset))) #define HASH_MAX_PROBES 15 /* max entries to probe before rehashing */ +#define HASH_MAX_POWER 12 /* max power of 2 to create sub-tables */ /* return a hash layer with 2**power empty entries */ static struct hash_subtable * @@ -82,7 +90,7 @@ hash_subtable_new (Hmap *h, int32 power, int32 used) } static void -init_sizes (int64 hint, int32 *init_power, int32 *max_power) +init_sizes (int64 hint, int32 *init_power) { int32 log = 0; int32 i; @@ -98,24 +106,20 @@ init_sizes (int64 hint, int32 *init_power, int32 *max_power) } else { *init_power = 12; } - *max_power = 12; } static void hash_init (Hmap *h, int32 datasize, int64 hint) { int32 init_power; - int32 max_power; if(datasize < sizeof (void *)) datasize = sizeof (void *); datasize = runtime·rnd(datasize, sizeof (void *)); - init_sizes (hint, &init_power, &max_power); + init_sizes (hint, &init_power); h->datasize = datasize; - h->max_power = max_power; assert (h->datasize == datasize); - assert (h->max_power == max_power); - assert (sizeof (void *) <= h->datasize || h->max_power == 255); + assert (sizeof (void *) <= h->datasize); h->count = 0; h->changes = 0; h->st = hash_subtable_new (h, init_power, 0); @@ -253,7 +257,8 @@ hash_grow (MapType *t, Hmap *h, struct hash_subtable **pst, int32 flags) used++; } } - free (old_st); + if (h->flag & CanFreeTable) + free (old_st); } static int32 @@ -266,6 +271,7 @@ hash_lookup (MapType *t, Hmap *h, void *data, void **pres) hash_hash_t e_hash; struct hash_entry *e; struct hash_entry *end_e; + void *key; bool eq; hash = h->hash0; @@ -290,7 +296,10 @@ hash_lookup (MapType *t, Hmap *h, void *data, void **pres) e = HASH_OFFSET (e, elemsize); } while (e != end_e && ((e_hash = e->hash) ^ hash) < HASH_SUBHASH) { - if (HASH_DATA_EQ (eq, t, h, data, e->data)) { /* a match */ + key = e->data; + if (h->flag & IndirectKey) + key = *(void**)e->data; + if (HASH_DATA_EQ (eq, t, h, data, key)) { /* a match */ *pres = e->data; return (1); } @@ -312,6 +321,7 @@ hash_remove (MapType *t, Hmap *h, void *data) struct hash_entry *e; struct hash_entry *end_e; bool eq; + void *key; hash = h->hash0; (*t->key->alg->hash) (&hash, t->key->size, data); @@ -335,8 +345,20 @@ hash_remove (MapType *t, Hmap *h, void *data) e = HASH_OFFSET (e, elemsize); } while (e != end_e && ((e_hash = e->hash) ^ hash) < HASH_SUBHASH) { - if (HASH_DATA_EQ (eq, t, h, data, e->data)) { /* a match */ - if (h->indirectval) + key = e->data; + if (h->flag & IndirectKey) + key = *(void**)e->data; + if (HASH_DATA_EQ (eq, t, h, data, key)) { /* a match */ + // Free key if indirect, but only if reflect can't be + // holding a pointer to it. Deletions are rare, + // indirect (large) keys are rare, reflect on maps + // is rare. So in the rare, rare, rare case of deleting + // an indirect key from a map that has been reflected on, + // we leave the key for garbage collection instead of + // freeing it here. + if (h->flag & CanFreeKey) + free (key); + if (h->flag & IndirectVal) free (*(void**)((byte*)e->data + h->valoff)); hash_remove_n (st, e, 1); h->count--; @@ -385,8 +407,12 @@ hash_insert_internal (MapType *t, struct hash_subtable **pst, int32 flags, hash_ struct hash_entry *ins_e = e; int32 ins_i = i; hash_hash_t ins_e_hash; + void *key; while (ins_e != end_e && ((e_hash = ins_e->hash) ^ hash) < HASH_SUBHASH) { - if (HASH_DATA_EQ (eq, t, h, data, ins_e->data)) { /* a match */ + key = ins_e->data; + if (h->flag & IndirectKey) + key = *(void**)key; + if (HASH_DATA_EQ (eq, t, h, data, key)) { /* a match */ *pres = ins_e->data; return (1); } @@ -423,7 +449,7 @@ hash_insert_internal (MapType *t, struct hash_subtable **pst, int32 flags, hash_ return (0); } h->changes++; - if (st->power < h->max_power) { + if (st->power < HASH_MAX_POWER) { hash_grow (t, h, pst, flags); } else { hash_conv (t, h, st, flags, hash, start_e); @@ -606,7 +632,7 @@ hash_iter_init (MapType *t, Hmap *h, struct hash_iter *it) } static void -clean_st (struct hash_subtable *st, int32 *slots, int32 *used) +clean_st (Hmap *h, struct hash_subtable *st, int32 *slots, int32 *used) { int32 elemsize = st->datasize + offsetof (struct hash_entry, data[0]); struct hash_entry *e = st->entry; @@ -617,13 +643,14 @@ clean_st (struct hash_subtable *st, int32 *slots, int32 *used) while (e <= last) { hash_hash_t hash = e->hash; if ((hash & HASH_MASK) == HASH_SUBHASH) { - clean_st (*(struct hash_subtable **)e->data, slots, used); + clean_st (h, *(struct hash_subtable **)e->data, slots, used); } else { lused += (hash != HASH_NIL); } e = HASH_OFFSET (e, elemsize); } - free (st); + if (h->flag & CanFreeTable) + free (st); *slots += lslots; *used += lused; } @@ -634,7 +661,7 @@ hash_destroy (Hmap *h) int32 slots = 0; int32 used = 0; - clean_st (h->st, &slots, &used); + clean_st (h, h->st, &slots, &used); free (h); } @@ -677,20 +704,23 @@ hash_visit (Hmap *h, void (*data_visit) (void *arg, int32 level, void *data), vo /// interfaces to go runtime // -// hash requires < 256 bytes of data (key+value) stored inline. -// Only basic types can be key - biggest is complex128 (16 bytes). -// Leave some room to grow, just in case. -enum { - MaxValsize = 256 - 64 -}; +static void** +hash_valptr(Hmap *h, void *p) +{ + p = (byte*)p + h->valoff; + if(h->flag & IndirectVal) + p = *(void**)p; + return p; +} + static void** -hash_indirect(Hmap *h, void *p) +hash_keyptr(Hmap *h, void *p) { - if(h->indirectval) + if(h->flag & IndirectKey) p = *(void**)p; return p; -} +} static int32 debug = 0; @@ -699,8 +729,8 @@ Hmap* runtime·makemap_c(MapType *typ, int64 hint) { Hmap *h; - int32 valsize_in_hash; Type *key, *val; + uintptr ksize, vsize; key = typ->key; val = typ->elem; @@ -712,19 +742,29 @@ runtime·makemap_c(MapType *typ, int64 hint) runtime·throw("runtime.makemap: unsupported map key type"); h = runtime·mal(sizeof(*h)); + h->flag |= CanFreeTable; /* until reflect gets involved, free is okay */ + + ksize = runtime·rnd(key->size, sizeof(void*)); + vsize = runtime·rnd(val->size, sizeof(void*)); + if(ksize > MaxData || vsize > MaxData || ksize+vsize > MaxData) { + // Either key is too big, or value is, or combined they are. + // Prefer to keep the key if possible, because we look at + // keys more often than values. + if(ksize > MaxData - sizeof(void*)) { + // No choice but to indirect the key. + h->flag |= IndirectKey; + h->flag |= CanFreeKey; /* until reflect gets involved, free is okay */ + ksize = sizeof(void*); + } + if(vsize > MaxData - ksize) { + // Have to indirect the value. + h->flag |= IndirectVal; + vsize = sizeof(void*); + } + } - valsize_in_hash = val->size; - if (val->size > MaxValsize) { - h->indirectval = 1; - valsize_in_hash = sizeof(void*); - } - - // Align value inside data so that mark-sweep gc can find it. - h->valoff = key->size; - if(valsize_in_hash >= sizeof(void*)) - h->valoff = runtime·rnd(key->size, sizeof(void*)); - - hash_init(h, h->valoff+valsize_in_hash, hint); + h->valoff = ksize; + hash_init(h, ksize+vsize, hint); // these calculations are compiler dependent. // figure out offsets of map call arguments. @@ -773,7 +813,7 @@ runtime·mapaccess(MapType *t, Hmap *h, byte *ak, byte *av, bool *pres) res = nil; if(hash_lookup(t, h, ak, (void**)&res)) { *pres = true; - elem->alg->copy(elem->size, av, hash_indirect(h, res+h->valoff)); + elem->alg->copy(elem->size, av, hash_valptr(h, res)); } else { *pres = false; elem->alg->copy(elem->size, av, nil); @@ -877,10 +917,14 @@ runtime·mapassign(MapType *t, Hmap *h, byte *ak, byte *av) res = nil; hit = hash_insert(t, h, ak, (void**)&res); - if(!hit && h->indirectval) - *(void**)(res+h->valoff) = runtime·mal(t->elem->size); - t->key->alg->copy(t->key->size, res, ak); - t->elem->alg->copy(t->elem->size, hash_indirect(h, res+h->valoff), av); + if(!hit) { + if(h->flag & IndirectKey) + *(void**)res = runtime·mal(t->key->size); + if(h->flag & IndirectVal) + *(void**)(res+h->valoff) = runtime·mal(t->elem->size); + } + t->key->alg->copy(t->key->size, hash_keyptr(h, res), ak); + t->elem->alg->copy(t->elem->size, hash_valptr(h, res), av); if(debug) { runtime·prints("mapassign: map="); @@ -985,6 +1029,22 @@ runtime·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it) void reflect·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it) { + uint8 flag; + + if(h != nil && t->key->size > sizeof(void*)) { + // reflect·mapiterkey returns pointers to key data, + // and reflect holds them, so we cannot free key data + // eagerly anymore. Updating h->flag now is racy, + // but it's okay because this is the only possible store + // after creation. + flag = h->flag; + if(flag & IndirectKey) + flag &= ~CanFreeKey; + else + flag &= ~CanFreeTable; + h->flag = flag; + } + it = runtime·mal(sizeof *it); FLUSH(&it); runtime·mapiterinit(t, h, it); @@ -1032,7 +1092,7 @@ runtime·mapiter1(struct hash_iter *it, ...) runtime·throw("runtime.mapiter1: key:val nil pointer"); key = it->t->key; - key->alg->copy(key->size, ak, res); + key->alg->copy(key->size, ak, hash_keyptr(h, res)); if(debug) { runtime·prints("mapiter2: iter="); @@ -1053,7 +1113,7 @@ runtime·mapiterkey(struct hash_iter *it, void *ak) if(res == nil) return false; key = it->t->key; - key->alg->copy(key->size, ak, res); + key->alg->copy(key->size, ak, hash_keyptr(it->h, res)); return true; } @@ -1076,6 +1136,7 @@ reflect·mapiterkey(struct hash_iter *it, uintptr key, bool ok) } else { tkey = it->t->key; key = 0; + res = (byte*)hash_keyptr(it->h, res); if(tkey->size <= sizeof(key)) tkey->alg->copy(tkey->size, (byte*)&key, res); else @@ -1117,8 +1178,8 @@ runtime·mapiter2(struct hash_iter *it, ...) runtime·throw("runtime.mapiter2: key:val nil pointer"); h = it->h; - t->key->alg->copy(t->key->size, ak, res); - t->elem->alg->copy(t->elem->size, av, hash_indirect(h, res+h->valoff)); + t->key->alg->copy(t->key->size, ak, hash_keyptr(h, res)); + t->elem->alg->copy(t->elem->size, av, hash_valptr(h, res)); if(debug) { runtime·prints("mapiter2: iter="); diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc index fbdd6bb02..9ae3a9d61 100644 --- a/src/pkg/runtime/malloc.goc +++ b/src/pkg/runtime/malloc.goc @@ -154,6 +154,7 @@ runtime·free(void *v) c->local_by_size[sizeclass].nfree++; runtime·MCache_Free(c, v, sizeclass, size); } + c->local_nfree++; c->local_alloc -= size; if(prof) runtime·MProf_Free(v, size); diff --git a/src/pkg/runtime/mfinal.c b/src/pkg/runtime/mfinal.c index c6f2b5421..1fa5ea401 100644 --- a/src/pkg/runtime/mfinal.c +++ b/src/pkg/runtime/mfinal.c @@ -150,8 +150,7 @@ runtime·addfinalizer(void *p, void (*f)(void*), int32 nret) tab = TAB(p); runtime·lock(tab); if(f == nil) { - if(lookfintab(tab, p, true, nil)) - runtime·setblockspecial(p, false); + lookfintab(tab, p, true, nil); runtime·unlock(tab); return true; } diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c index e043864c1..e8fb266f4 100644 --- a/src/pkg/runtime/mgc0.c +++ b/src/pkg/runtime/mgc0.c @@ -1066,7 +1066,6 @@ runfinq(void) framecap = framesz; } *(void**)frame = f->arg; - runtime·setblockspecial(f->arg, false); reflect·call((byte*)f->fn, frame, sizeof(uintptr) + f->nret); f->fn = nil; f->arg = nil; diff --git a/src/pkg/runtime/softfloat_arm.c b/src/pkg/runtime/softfloat_arm.c index fbe0b0413..bd73cb15b 100644 --- a/src/pkg/runtime/softfloat_arm.c +++ b/src/pkg/runtime/softfloat_arm.c @@ -9,10 +9,10 @@ #include "runtime.h" #define CPSR 14 -#define FLAGS_N (1 << 31) -#define FLAGS_Z (1 << 30) -#define FLAGS_C (1 << 29) -#define FLAGS_V (1 << 28) +#define FLAGS_N (1U << 31) +#define FLAGS_Z (1U << 30) +#define FLAGS_C (1U << 29) +#define FLAGS_V (1U << 28) void runtime·abort(void); void math·sqrtC(uint64, uint64*); @@ -86,12 +86,24 @@ fstatus(bool nan, int32 cmp) return FLAGS_C; } +// conditions array record the required CPSR cond field for the +// first 5 pairs of conditional execution opcodes +// higher 4 bits are must set, lower 4 bits are must clear +static const uint8 conditions[10/2] = { + [0/2] = (FLAGS_Z >> 24) | 0, // 0: EQ (Z set), 1: NE (Z clear) + [2/2] = (FLAGS_C >> 24) | 0, // 2: CS/HS (C set), 3: CC/LO (C clear) + [4/2] = (FLAGS_N >> 24) | 0, // 4: MI (N set), 5: PL (N clear) + [6/2] = (FLAGS_V >> 24) | 0, // 6: VS (V set), 7: VC (V clear) + [8/2] = (FLAGS_C >> 24) | + (FLAGS_Z >> 28), // 8: HI (C set and Z clear), 9: LS (C clear and Z set) +}; + // returns number of words that the fp instruction // is occupying, 0 if next instruction isn't float. static uint32 stepflt(uint32 *pc, uint32 *regs) { - uint32 i, regd, regm, regn; + uint32 i, opc, regd, regm, regn, cpsr; int32 delta; uint32 *addr; uint64 uval; @@ -102,8 +114,49 @@ stepflt(uint32 *pc, uint32 *regs) i = *pc; if(trace) - runtime·printf("stepflt %p %x\n", pc, i); + runtime·printf("stepflt %p %x (cpsr %x)\n", pc, i, regs[CPSR] >> 28); + + opc = i >> 28; + if(opc == 14) // common case first + goto execute; + cpsr = regs[CPSR] >> 28; + switch(opc) { + case 0: case 1: case 2: case 3: case 4: + case 5: case 6: case 7: case 8: case 9: + if(((cpsr & (conditions[opc/2] >> 4)) == (conditions[opc/2] >> 4)) && + ((cpsr & (conditions[opc/2] & 0xf)) == 0)) { + if(opc & 1) return 1; + } else { + if(!(opc & 1)) return 1; + } + break; + case 10: // GE (N == V) + case 11: // LT (N != V) + if((cpsr & (FLAGS_N >> 28)) == (cpsr & (FLAGS_V >> 28))) { + if(opc & 1) return 1; + } else { + if(!(opc & 1)) return 1; + } + break; + case 12: // GT (N == V and Z == 0) + case 13: // LE (N != V or Z == 1) + if((cpsr & (FLAGS_N >> 28)) == (cpsr & (FLAGS_V >> 28)) && + (cpsr & (FLAGS_Z >> 28)) == 0) { + if(opc & 1) return 1; + } else { + if(!(opc & 1)) return 1; + } + break; + case 14: // AL + break; + case 15: // shouldn't happen + return 0; + } + if(trace) + runtime·printf("conditional %x (cpsr %x) pass\n", opc, cpsr); + i = (0xeU << 28) | (i & 0xfffffff); +execute: // special cases if((i&0xfffff000) == 0xe59fb000) { // load r11 from pc-relative address. diff --git a/src/pkg/runtime/time.goc b/src/pkg/runtime/time.goc index a6b835247..b18902f00 100644 --- a/src/pkg/runtime/time.goc +++ b/src/pkg/runtime/time.goc @@ -61,8 +61,11 @@ runtime·tsleep(int64 ns) { Timer t; - if(ns <= 0) + if(ns <= 0) { + g->status = Grunning; + g->waitreason = nil; return; + } t.when = runtime·nanotime() + ns; t.period = 0; diff --git a/src/pkg/runtime/vlrt_arm.c b/src/pkg/runtime/vlrt_arm.c index 50f33710b..ab8050177 100644 --- a/src/pkg/runtime/vlrt_arm.c +++ b/src/pkg/runtime/vlrt_arm.c @@ -197,12 +197,14 @@ void runtime·int64tofloat64(Vlong y, double d) { d = _v2d(y); + USED(&d); // FLUSH } void runtime·uint64tofloat64(Vlong y, double d) { d = _ul2d(y.hi)*4294967296. + _ul2d(y.lo); + USED(&d); // FLUSH } static void |