1 files changed, 134 insertions, 312 deletions

diff --git a/src/pkg/runtime/hashmap.c b/src/pkg/runtime/hashmap.c
index 892f0a170..6d2ab2168 100644
--- a/src/pkg/runtime/hashmap.c
+++ b/src/pkg/runtime/hashmap.c
@@ -5,9 +5,9 @@
 #include "runtime.h"
 #include "arch_GOARCH.h"
 #include "malloc.h"
-#include "hashmap.h"
 #include "type.h"
 #include "race.h"
+#include "../../cmd/ld/textflag.h"
 
 // This file contains the implementation of Go's map type.
 //
@@ -74,6 +74,8 @@
 typedef struct Bucket Bucket;
 struct Bucket
 {
+	// Note: the format of the Bucket is encoded in ../../cmd/gc/reflect.c and
+	// ../reflect/type.go.  Don't change this structure without also changing that code!
 	uint8  tophash[BUCKETSIZE]; // top 8 bits of hash of each entry (0 = empty)
 	Bucket *overflow;           // overflow bucket, if any
 	byte   data[1];             // BUCKETSIZE keys followed by BUCKETSIZE values
@@ -89,19 +91,18 @@ struct Bucket
 #define evacuated(b) (((uintptr)(b)->overflow & 1) != 0)
 #define overflowptr(b) ((Bucket*)((uintptr)(b)->overflow & ~(uintptr)1))
 
-// Initialize bucket to the empty state.  This only works if BUCKETSIZE==8!
-#define clearbucket(b) { *(uint64*)((b)->tophash) = 0; (b)->overflow = nil; }
-
 struct Hmap
 {
+	// Note: the format of the Hmap is encoded in ../../cmd/gc/reflect.c and
+	// ../reflect/type.go.  Don't change this structure without also changing that code!
 	uintgo  count;        // # live cells == size of map.  Must be first (used by len() builtin)
 	uint32  flags;
+	uint32  hash0;        // hash seed
 	uint8   B;            // log_2 of # of buckets (can hold up to LOAD * 2^B items)
 	uint8   keysize;      // key size in bytes
 	uint8   valuesize;    // value size in bytes
 	uint16  bucketsize;   // bucket size in bytes
 
-	uintptr hash0;        // hash seed
 	byte    *buckets;     // array of 2^B Buckets. may be nil if count==0.
 	byte    *oldbuckets;  // previous bucket array of half the size, non-nil only when growing
 	uintptr nevacuate;    // progress counter for evacuation (buckets less than this have been evacuated)
@@ -114,8 +115,6 @@ enum
 	IndirectValue = 2,  // storing pointers to values
 	Iterator = 4,       // there may be an iterator using buckets
 	OldIterator = 8,    // there may be an iterator using oldbuckets
-	CanFreeBucket = 16, // ok to free buckets
-	CanFreeKey = 32,    // keys are indirect and ok to free keys
 };
 
 // Macros for dereferencing indirect keys
@@ -208,17 +207,15 @@ hash_init(MapType *t, Hmap *h, uint32 hint)
 {
 	uint8 B;
 	byte *buckets;
-	uintptr i;
 	uintptr keysize, valuesize, bucketsize;
 	uint8 flags;
-	Bucket *b;
 
-	flags = CanFreeBucket;
+	flags = 0;
 
 	// figure out how big we have to make everything
 	keysize = t->key->size;
 	if(keysize > MAXKEYSIZE) {
-		flags |= IndirectKey | CanFreeKey;
+		flags |= IndirectKey;
 		keysize = sizeof(byte*);
 	}
 	valuesize = t->elem->size;
@@ -240,8 +237,6 @@ hash_init(MapType *t, Hmap *h, uint32 hint)
 		runtime·throw("value size not a multiple of value align");
 	if(BUCKETSIZE < 8)
 		runtime·throw("bucketsize too small for proper alignment");
-	if(BUCKETSIZE != 8)
-		runtime·throw("must redo clearbucket");
 	if(sizeof(void*) == 4 && t->key->align > 4)
 		runtime·throw("need padding in bucket (key)");
 	if(sizeof(void*) == 4 && t->elem->align > 4)
@@ -259,16 +254,10 @@ hash_init(MapType *t, Hmap *h, uint32 hint)
 		// done lazily later.
 		buckets = nil;
 	} else {
-		buckets = runtime·mallocgc(bucketsize << B, 0, 1, 0);
-		for(i = 0; i < (uintptr)1 << B; i++) {
-			b = (Bucket*)(buckets + i * bucketsize);
-			clearbucket(b);
-		}
+		buckets = runtime·cnewarray(t->bucket, (uintptr)1 << B);
 	}
 
 	// initialize Hmap
-	// Note: we save all these stores to the end so gciter doesn't see
-	// a partially initialized map.
 	h->count = 0;
 	h->B = B;
 	h->flags = flags;
@@ -299,19 +288,18 @@ evacuate(MapType *t, Hmap *h, uintptr oldbucket)
 	uintptr i;
 	byte *k, *v;
 	byte *xk, *yk, *xv, *yv;
-	byte *ob;
+	uint8 top;
+	bool eq;
 
 	b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize);
 	newbit = (uintptr)1 << (h->B - 1);
 
 	if(!evacuated(b)) {
 		// TODO: reuse overflow buckets instead of using new ones, if there
-		// is no iterator using the old buckets.  (If CanFreeBuckets and !OldIterator.)
+		// is no iterator using the old buckets.  (If !OldIterator.)
 
 		x = (Bucket*)(h->buckets + oldbucket * h->bucketsize);
 		y = (Bucket*)(h->buckets + (oldbucket + newbit) * h->bucketsize);
-		clearbucket(x);
-		clearbucket(y);
 		xi = 0;
 		yi = 0;
 		xk = x->data;
@@ -320,25 +308,49 @@ evacuate(MapType *t, Hmap *h, uintptr oldbucket)
 		yv = yk + h->keysize * BUCKETSIZE;
 		do {
 			for(i = 0, k = b->data, v = k + h->keysize * BUCKETSIZE; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) {
-				if(b->tophash[i] == 0)
+				top = b->tophash[i];
+				if(top == 0)
 					continue;
+
+				// Compute hash to make our evacuation decision (whether we need
+				// to send this key/value to bucket x or bucket y).
 				hash = h->hash0;
 				t->key->alg->hash(&hash, t->key->size, IK(h, k));
-				// NOTE: if key != key, then this hash could be (and probably will be)
-				// entirely different from the old hash.  We effectively only update
-				// the B'th bit of the hash in this case.
+				if((h->flags & Iterator) != 0) {
+					t->key->alg->equal(&eq, t->key->size, IK(h, k), IK(h, k));
+					if(!eq) {
+						// If key != key (NaNs), then the hash could be (and probably
+						// will be) entirely different from the old hash.  Moreover,
+						// it isn't reproducible.  Reproducibility is required in the
+						// presence of iterators, as our evacuation decision must
+						// match whatever decision the iterator made.
+						// Fortunately, we have the freedom to send these keys either
+						// way.  Also, tophash is meaningless for these kinds of keys.
+						// We let the low bit of tophash drive the evacuation decision.
+						// We recompute a new random tophash for the next level so
+						// these keys will get evenly distributed across all buckets
+						// after multiple grows.
+						if((top & 1) != 0)
+							hash |= newbit;
+						else
+							hash &= ~newbit;
+						top = hash >> (8*sizeof(uintptr)-8);
+						if(top == 0)
+							top = 1;
+					}
+				}
+
 				if((hash & newbit) == 0) {
 					if(xi == BUCKETSIZE) {
 						if(checkgc) mstats.next_gc = mstats.heap_alloc;
-						newx = runtime·mallocgc(h->bucketsize, 0, 1, 0);
-						clearbucket(newx);
+						newx = runtime·cnew(t->bucket);
 						x->overflow = newx;
 						x = newx;
 						xi = 0;
 						xk = x->data;
 						xv = xk + h->keysize * BUCKETSIZE;
 					}
-					x->tophash[xi] = b->tophash[i];
+					x->tophash[xi] = top;
 					if((h->flags & IndirectKey) != 0) {
 						*(byte**)xk = *(byte**)k;               // copy pointer
 					} else {
@@ -355,15 +367,14 @@ evacuate(MapType *t, Hmap *h, uintptr oldbucket)
 				} else {
 					if(yi == BUCKETSIZE) {
 						if(checkgc) mstats.next_gc = mstats.heap_alloc;
-						newy = runtime·mallocgc(h->bucketsize, 0, 1, 0);
-						clearbucket(newy);
+						newy = runtime·cnew(t->bucket);
 						y->overflow = newy;
 						y = newy;
 						yi = 0;
 						yk = y->data;
 						yv = yk + h->keysize * BUCKETSIZE;
 					}
-					y->tophash[yi] = b->tophash[i];
+					y->tophash[yi] = top;
 					if((h->flags & IndirectKey) != 0) {
 						*(byte**)yk = *(byte**)k;
 					} else {
@@ -388,35 +399,18 @@ evacuate(MapType *t, Hmap *h, uintptr oldbucket)
 			b = nextb;
 		} while(b != nil);
 
-		// Free old overflow buckets as much as we can.
-		if((h->flags & OldIterator) == 0) {
-			b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize);
-			if((h->flags & CanFreeBucket) != 0) {
-				while((nextb = overflowptr(b)) != nil) {
-					b->overflow = nextb->overflow;
-					runtime·free(nextb);
-				}
-			} else {
-				// can't explicitly free overflow buckets, but at least
-				// we can unlink them.
-				b->overflow = (Bucket*)1;
-			}
-		}
+		// Unlink the overflow buckets to help GC.
+		b = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize);
+		if((h->flags & OldIterator) == 0)
+			b->overflow = (Bucket*)1;
 	}
 
 	// advance evacuation mark
 	if(oldbucket == h->nevacuate) {
 		h->nevacuate = oldbucket + 1;
-		if(oldbucket + 1 == newbit) { // newbit == # of oldbuckets
+		if(oldbucket + 1 == newbit) // newbit == # of oldbuckets
 			// free main bucket array
-			if((h->flags & (OldIterator | CanFreeBucket)) == CanFreeBucket) {
-				ob = h->oldbuckets;
-				h->oldbuckets = nil;
-				runtime·free(ob);
-			} else {
-				h->oldbuckets = nil;
-			}
-		}
+			h->oldbuckets = nil;
 	}
 	if(docheck)
 		check(t, h);
@@ -451,14 +445,10 @@ hash_grow(MapType *t, Hmap *h)
 	old_buckets = h->buckets;
 	// NOTE: this could be a big malloc, but since we don't need zeroing it is probably fast.
 	if(checkgc) mstats.next_gc = mstats.heap_alloc;
-	new_buckets = runtime·mallocgc((uintptr)h->bucketsize << (h->B + 1), 0, 1, 0);
+	new_buckets = runtime·cnewarray(t->bucket, (uintptr)1 << (h->B + 1));
 	flags = (h->flags & ~(Iterator | OldIterator));
-	if((h->flags & Iterator) != 0) {
+	if((h->flags & Iterator) != 0)
 		flags |= OldIterator;
-		// We can't free indirect keys any more, as
-		// they are potentially aliased across buckets.
-		flags &= ~CanFreeKey;
-	}
 
 	// commit the grow (atomic wrt gc)
 	h->B++;
@@ -524,6 +514,7 @@ hash_lookup(MapType *t, Hmap *h, byte **keyp)
 }
 
 // When an item is not found, fast versions return a pointer to this zeroed memory.
+#pragma dataflag RODATA
 static uint8 empty_value[MAXVALUESIZE];
 
 // Specialized versions of mapaccess1 for specific types.
@@ -532,48 +523,65 @@ static uint8 empty_value[MAXVALUESIZE];
 #define HASH_LOOKUP2 runtime·mapaccess2_fast32
 #define KEYTYPE uint32
 #define HASHFUNC runtime·algarray[AMEM32].hash
-#define EQFUNC(x,y) ((x) == (y))
-#define EQMAYBE(x,y) ((x) == (y))
-#define HASMAYBE false
-#define QUICKEQ(x) true
+#define FASTKEY(x) true
+#define QUICK_NE(x,y) ((x) != (y))
+#define QUICK_EQ(x,y) true
+#define SLOW_EQ(x,y) true
+#define MAYBE_EQ(x,y) true
 #include "hashmap_fast.c"
 
 #undef HASH_LOOKUP1
 #undef HASH_LOOKUP2
 #undef KEYTYPE
 #undef HASHFUNC
-#undef EQFUNC
-#undef EQMAYBE
-#undef HASMAYBE
-#undef QUICKEQ
+#undef FASTKEY
+#undef QUICK_NE
+#undef QUICK_EQ
+#undef SLOW_EQ
+#undef MAYBE_EQ
 
 #define HASH_LOOKUP1 runtime·mapaccess1_fast64
 #define HASH_LOOKUP2 runtime·mapaccess2_fast64
 #define KEYTYPE uint64
 #define HASHFUNC runtime·algarray[AMEM64].hash
-#define EQFUNC(x,y) ((x) == (y))
-#define EQMAYBE(x,y) ((x) == (y))
-#define HASMAYBE false
-#define QUICKEQ(x) true
+#define FASTKEY(x) true
+#define QUICK_NE(x,y) ((x) != (y))
+#define QUICK_EQ(x,y) true
+#define SLOW_EQ(x,y) true
+#define MAYBE_EQ(x,y) true
 #include "hashmap_fast.c"
 
 #undef HASH_LOOKUP1
 #undef HASH_LOOKUP2
 #undef KEYTYPE
 #undef HASHFUNC
-#undef EQFUNC
-#undef EQMAYBE
-#undef HASMAYBE
-#undef QUICKEQ
+#undef FASTKEY
+#undef QUICK_NE
+#undef QUICK_EQ
+#undef SLOW_EQ
+#undef MAYBE_EQ
+
+#ifdef GOARCH_amd64
+#define CHECKTYPE uint64
+#endif
+#ifdef GOARCH_386
+#define CHECKTYPE uint32
+#endif
+#ifdef GOARCH_arm
+// can't use uint32 on arm because our loads aren't aligned.
+// TODO: use uint32 for arm v6+?
+#define CHECKTYPE uint8
+#endif
 
 #define HASH_LOOKUP1 runtime·mapaccess1_faststr
 #define HASH_LOOKUP2 runtime·mapaccess2_faststr
 #define KEYTYPE String
 #define HASHFUNC runtime·algarray[ASTRING].hash
-#define EQFUNC(x,y) ((x).len == (y).len && ((x).str == (y).str || runtime·memeq((x).str, (y).str, (x).len)))
-#define EQMAYBE(x,y) ((x).len == (y).len)
-#define HASMAYBE true
-#define QUICKEQ(x) ((x).len < 32)
+#define FASTKEY(x) ((x).len < 32)
+#define QUICK_NE(x,y) ((x).len != (y).len)
+#define QUICK_EQ(x,y) ((x).str == (y).str)
+#define SLOW_EQ(x,y) runtime·memeq((x).str, (y).str, (x).len)
+#define MAYBE_EQ(x,y) (*(CHECKTYPE*)(x).str == *(CHECKTYPE*)(y).str && *(CHECKTYPE*)((x).str + (x).len - sizeof(CHECKTYPE)) == *(CHECKTYPE*)((y).str + (x).len - sizeof(CHECKTYPE)))
 #include "hashmap_fast.c"
 
 static void
@@ -595,11 +603,8 @@ hash_insert(MapType *t, Hmap *h, void *key, void *value)
 		check(t, h);
 	hash = h->hash0;
 	t->key->alg->hash(&hash, t->key->size, key);
-	if(h->buckets == nil) {
-		h->buckets = runtime·mallocgc(h->bucketsize, 0, 1, 0);
-		b = (Bucket*)(h->buckets);
-		clearbucket(b);
-	}
+	if(h->buckets == nil)
+		h->buckets = runtime·cnewarray(t->bucket, 1);
 
  again:
 	bucket = hash & (((uintptr)1 << h->B) - 1);
@@ -609,7 +614,7 @@ hash_insert(MapType *t, Hmap *h, void *key, void *value)
 	top = hash >> (sizeof(uintptr)*8 - 8);
 	if(top == 0)
 		top = 1;
-	inserti = 0;
+	inserti = nil;
 	insertk = nil;
 	insertv = nil;
 	while(true) {
@@ -646,8 +651,7 @@ hash_insert(MapType *t, Hmap *h, void *key, void *value)
 	if(inserti == nil) {
 		// all current buckets are full, allocate a new one.
 		if(checkgc) mstats.next_gc = mstats.heap_alloc;
-		newb = runtime·mallocgc(h->bucketsize, 0, 1, 0);
-		clearbucket(newb);
+		newb = runtime·cnew(t->bucket);
 		b->overflow = newb;
 		inserti = newb->tophash;
 		insertk = newb->data;
@@ -657,13 +661,13 @@ hash_insert(MapType *t, Hmap *h, void *key, void *value)
 	// store new key/value at insert position
 	if((h->flags & IndirectKey) != 0) {
 		if(checkgc) mstats.next_gc = mstats.heap_alloc;
-		kmem = runtime·mallocgc(t->key->size, 0, 1, 0);
+		kmem = runtime·cnew(t->key);
 		*(byte**)insertk = kmem;
 		insertk = kmem;
 	}
 	if((h->flags & IndirectValue) != 0) {
 		if(checkgc) mstats.next_gc = mstats.heap_alloc;
-		vmem = runtime·mallocgc(t->elem->size, 0, 1, 0);
+		vmem = runtime·cnew(t->elem);
 		*(byte**)insertv = vmem;
 		insertv = vmem;
 	}
@@ -707,22 +711,20 @@ hash_remove(MapType *t, Hmap *h, void *key)
 			if(!eq)
 				continue;
 
-			if((h->flags & CanFreeKey) != 0) {
-				k = *(byte**)k;
+			if((h->flags & IndirectKey) != 0) {
+				*(byte**)k = nil;
+			} else {
+				t->key->alg->copy(t->key->size, k, nil);
 			}
 			if((h->flags & IndirectValue) != 0) {
-				v = *(byte**)v;
+				*(byte**)v = nil;
+			} else {
+				t->elem->alg->copy(t->elem->size, v, nil);
 			}
 
 			b->tophash[i] = 0;
 			h->count--;
 			
-			if((h->flags & CanFreeKey) != 0) {
-				runtime·free(k);
-			}
-			if((h->flags & IndirectValue) != 0) {
-				runtime·free(v);
-			}
 			// TODO: consolidate buckets if they are mostly empty
 			// can only consolidate if there are no live iterators at this size.
 			if(docheck)
@@ -785,7 +787,7 @@ hash_iter_init(MapType *t, Hmap *h, struct hash_iter *it)
 	it->bptr = nil;
 
 	// Remember we have an iterator.
-	// Can run concurrently with another hash_iter_init() and with reflect·mapiterinit().
+	// Can run concurrently with another hash_iter_init().
 	for(;;) {
 		old = h->flags;
 		if((old&(Iterator|OldIterator)) == (Iterator|OldIterator))
@@ -863,18 +865,12 @@ next:
 			if(check_bucket >= 0) {
 				// Special case: iterator was started during a grow and the
 				// grow is not done yet.  We're working on a bucket whose
-				// oldbucket has not been evacuated yet.  So we iterate
+				// oldbucket has not been evacuated yet.  So we're iterating
 				// through the oldbucket, skipping any keys that will go
 				// to the other new bucket (each oldbucket expands to two
 				// buckets during a grow).
 				t->key->alg->equal(&eq, t->key->size, IK(h, k), IK(h, k));
-				if(!eq) {
-					// Hash is meaningless if k != k (NaNs).  Return all
-					// NaNs during the first of the two new buckets.
-					if(bucket >= ((uintptr)1 << (it->B - 1))) {
-						continue;
-					}
-				} else {
+				if(eq) {
 					// If the item in the oldbucket is not destined for
 					// the current new bucket in the iteration, skip it.
 					hash = h->hash0;
@@ -882,6 +878,14 @@ next:
 					if((hash & (((uintptr)1 << it->B) - 1)) != check_bucket) {
 						continue;
 					}
+				} else {
+					// Hash isn't repeatable if k != k (NaNs).  We need a
+					// repeatable and randomish choice of which direction
+					// to send NaNs during evacuation.  We'll use the low
+					// bit of tophash to decide which way NaNs go.
+					if(check_bucket >> (it->B - 1) != (b->tophash[i] & 1)) {
+						continue;
+					}
 				}
 			}
 			if(!evacuated(b)) {
@@ -925,157 +929,6 @@ next:
 	goto next;
 }
 
-
-#define PHASE_BUCKETS      0
-#define PHASE_OLD_BUCKETS  1
-#define PHASE_TABLE        2
-#define PHASE_OLD_TABLE    3
-#define PHASE_DONE         4
-
-// Initialize the iterator.
-// Returns false if Hmap contains no pointers (in which case the iterator is not initialized).
-bool
-hash_gciter_init (Hmap *h, struct hash_gciter *it)
-{
-	// GC during map initialization or on an empty map.
-	if(h->buckets == nil)
-		return false;
-
-	it->h = h;
-	it->phase = PHASE_BUCKETS;
-	it->bucket = 0;
-	it->b = nil;
-
-	// TODO: finish evacuating oldbuckets so that we can collect
-	// oldbuckets?  We don't want to keep a partially evacuated
-	// table around forever, so each gc could make at least some
-	// evacuation progress.  Need to be careful about concurrent
-	// access if we do concurrent gc.  Even if not, we don't want
-	// to make the gc pause any longer than it has to be.
-
-	return true;
-}
-
-// Returns true and fills *data with internal structure/key/value data,
-// or returns false if the iterator has terminated.
-// Ugh, this interface is really annoying.  I want a callback fn!
-bool
-hash_gciter_next(struct hash_gciter *it, struct hash_gciter_data *data)
-{
-	Hmap *h;
-	uintptr bucket, oldbucket;
-	Bucket *b, *oldb;
-	uintptr i;
-	byte *k, *v;
-
-	h = it->h;
-	bucket = it->bucket;
-	b = it->b;
-	i = it->i;
-
-	data->st = nil;
-	data->key_data = nil;
-	data->val_data = nil;
-	data->indirectkey = (h->flags & IndirectKey) != 0;
-	data->indirectval = (h->flags & IndirectValue) != 0;
-
-next:
-	switch (it->phase) {
-	case PHASE_BUCKETS:
-		if(b != nil) {
-			k = b->data + h->keysize * i;
-			v = b->data + h->keysize * BUCKETSIZE + h->valuesize * i;
-			for(; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) {
-				if(b->tophash[i] != 0) {
-					data->key_data = k;
-					data->val_data = v;
-					it->bucket = bucket;
-					it->b = b;
-					it->i = i + 1;
-					return true;
-				}
-			}
-			b = b->overflow;
-			if(b != nil) {
-				data->st = (byte*)b;
-				it->bucket = bucket;
-				it->b = b;
-				it->i = 0;
-				return true;
-			}
-		}
-		while(bucket < ((uintptr)1 << h->B)) {
-			if(h->oldbuckets != nil) {
-				oldbucket = bucket & (((uintptr)1 << (h->B - 1)) - 1);
-				oldb = (Bucket*)(h->oldbuckets + oldbucket * h->bucketsize);
-				if(!evacuated(oldb)) {
-					// new bucket isn't valid yet
-					bucket++;
-					continue;
-				}
-			}
-			b = (Bucket*)(h->buckets + bucket * h->bucketsize);
-			i = 0;
-			bucket++;
-			goto next;
-		}
-		it->phase = PHASE_OLD_BUCKETS;
-		bucket = 0;
-		b = nil;
-		goto next;
-	case PHASE_OLD_BUCKETS:
-		if(h->oldbuckets == nil) {
-			it->phase = PHASE_TABLE;
-			goto next;
-		}
-		if(b != nil) {
-			k = b->data + h->keysize * i;
-			v = b->data + h->keysize * BUCKETSIZE + h->valuesize * i;
-			for(; i < BUCKETSIZE; i++, k += h->keysize, v += h->valuesize) {
-				if(b->tophash[i] != 0) {
-					data->key_data = k;
-					data->val_data = v;
-					it->bucket = bucket;
-					it->b = b;
-					it->i = i + 1;
-					return true;
-				}
-			}
-			b = overflowptr(b);
-			if(b != nil) {
-				data->st = (byte*)b;
-				it->bucket = bucket;
-				it->b = b;
-				it->i = 0;
-				return true;
-			}
-		}
-		if(bucket < ((uintptr)1 << (h->B - 1))) {
-			b = (Bucket*)(h->oldbuckets + bucket * h->bucketsize);
-			bucket++;
-			i = 0;
-			goto next;
-		}
-		it->phase = PHASE_TABLE;
-		goto next;
-	case PHASE_TABLE:
-		it->phase = PHASE_OLD_TABLE;
-		data->st = h->buckets;
-		return true;
-	case PHASE_OLD_TABLE:
-		it->phase = PHASE_DONE;
-		if(h->oldbuckets != nil) {
-			data->st = h->oldbuckets;
-			return true;
-		} else {
-			goto next;
-		}
-	}
-	if(it->phase != PHASE_DONE)
-		runtime·throw("bad phase at done");
-	return false;
-}
-
 //
 /// interfaces to go runtime
 //
@@ -1094,6 +947,9 @@ runtime·makemap_c(MapType *typ, int64 hint)
 	Type *key;
 
 	key = typ->key;
+	
+	if(sizeof(Hmap) > 48)
+		runtime·panicstring("hmap too large");
 
 	if(hint < 0 || (int32)hint != hint)
 		runtime·panicstring("makemap: size out of range");
@@ -1101,15 +957,7 @@ runtime·makemap_c(MapType *typ, int64 hint)
 	if(key->alg->hash == runtime·nohash)
 		runtime·throw("runtime.makemap: unsupported map key type");
 
-	h = runtime·mal(sizeof(*h));
-
-	if(UseSpanType) {
-		if(false) {
-			runtime·printf("makemap %S: %p\n", *typ->string, h);
-		}
-		runtime·settype(h, (uintptr)typ | TypeInfo_Map);
-	}
-
+	h = runtime·cnew(typ->hmap);
 	hash_init(typ, h, hint);
 
 	// these calculations are compiler dependent.
@@ -1153,9 +1001,6 @@ runtime·mapaccess(MapType *t, Hmap *h, byte *ak, byte *av, bool *pres)
 		return;
 	}
 
-	if(runtime·gcwaiting)
-		runtime·gosched();
-
 	res = hash_lookup(t, h, &ak);
 
 	if(res != nil) {
@@ -1168,7 +1013,7 @@ runtime·mapaccess(MapType *t, Hmap *h, byte *ak, byte *av, bool *pres)
 }
 
 // mapaccess1(hmap *map[any]any, key any) (val any);
-#pragma textflag 7
+#pragma textflag NOSPLIT
 void
 runtime·mapaccess1(MapType *t, Hmap *h, ...)
 {
@@ -1200,7 +1045,7 @@ runtime·mapaccess1(MapType *t, Hmap *h, ...)
 }
 
 // mapaccess2(hmap *map[any]any, key any) (val any, pres bool);
-#pragma textflag 7
+#pragma textflag NOSPLIT
 void
 runtime·mapaccess2(MapType *t, Hmap *h, ...)
 {
@@ -1263,9 +1108,6 @@ runtime·mapassign(MapType *t, Hmap *h, byte *ak, byte *av)
 	if(h == nil)
 		runtime·panicstring("assignment to entry in nil map");
 
-	if(runtime·gcwaiting)
-		runtime·gosched();
-
 	if(av == nil) {
 		hash_remove(t, h, ak);
 	} else {
@@ -1278,13 +1120,16 @@ runtime·mapassign(MapType *t, Hmap *h, byte *ak, byte *av)
 		runtime·prints("; key=");
 		t->key->alg->print(t->key->size, ak);
 		runtime·prints("; val=");
-		t->elem->alg->print(t->elem->size, av);
+		if(av)
+			t->elem->alg->print(t->elem->size, av);
+		else
+			runtime·prints("nil");
 		runtime·prints("\n");
 	}
 }
 
 // mapassign1(mapType *type, hmap *map[any]any, key any, val any);
-#pragma textflag 7
+#pragma textflag NOSPLIT
 void
 runtime·mapassign1(MapType *t, Hmap *h, ...)
 {
@@ -1302,7 +1147,7 @@ runtime·mapassign1(MapType *t, Hmap *h, ...)
 }
 
 // mapdelete(mapType *type, hmap *map[any]any, key any)
-#pragma textflag 7
+#pragma textflag NOSPLIT
 void
 runtime·mapdelete(MapType *t, Hmap *h, ...)
 {
@@ -1355,7 +1200,7 @@ reflect·mapassign(MapType *t, Hmap *h, uintptr key, uintptr val, bool pres)
 void
 runtime·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it)
 {
-	if(h == nil) {
+	if(h == nil || h->count == 0) {
 		it->key = nil;
 		return;
 	}
@@ -1379,26 +1224,6 @@ runtime·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it)
 void
 reflect·mapiterinit(MapType *t, Hmap *h, struct hash_iter *it)
 {
-	uint32 old, new;
-
-	if(h != nil && t->key->size > sizeof(void*)) {
-		// reflect·mapiterkey returns pointers to key data,
-		// and reflect holds them, so we cannot free key data
-		// eagerly anymore.
-		// Can run concurrently with another reflect·mapiterinit() and with hash_iter_init().
-		for(;;) {
-			old = h->flags;
-			if(old & IndirectKey)
-				new = old & ~CanFreeKey;
-			else
-				new = old & ~CanFreeBucket;
-			if(new == old)
-				break;
-			if(runtime·cas(&h->flags, old, new))
-				break;
-		}
-	}
-
 	it = runtime·mal(sizeof *it);
 	FLUSH(&it);
 	runtime·mapiterinit(t, h, it);
@@ -1410,8 +1235,6 @@ runtime·mapiternext(struct hash_iter *it)
 {
 	if(raceenabled)
 		runtime·racereadpc(it->h, runtime·getcallerpc(&it), runtime·mapiternext);
-	if(runtime·gcwaiting)
-		runtime·gosched();
 
 	hash_next(it);
 	if(debug) {
@@ -1432,7 +1255,7 @@ reflect·mapiternext(struct hash_iter *it)
 }
 
 // mapiter1(hiter *any) (key any);
-#pragma textflag 7
+#pragma textflag NOSPLIT
 void
 runtime·mapiter1(struct hash_iter *it, ...)
 {
@@ -1484,12 +1307,8 @@ reflect·mapiterkey(struct hash_iter *it, uintptr key, bool ok)
 	key = 0;
 	ok = false;
 	res = it->key;
-	if(res == nil) {
-		key = 0;
-		ok = false;
-	} else {
+	if(res != nil) {
 		tkey = it->t->key;
-		key = 0;
 		if(tkey->size <= sizeof(key))
 			tkey->alg->copy(tkey->size, (byte*)&key, res);
 		else
@@ -1517,7 +1336,7 @@ reflect·maplen(Hmap *h, intgo len)
 }
 
 // mapiter2(hiter *any) (key any, val any);
-#pragma textflag 7
+#pragma textflag NOSPLIT
 void
 runtime·mapiter2(struct hash_iter *it, ...)
 {
@@ -1543,3 +1362,6 @@ runtime·mapiter2(struct hash_iter *it, ...)
 		runtime·prints("\n");
 	}
 }
+
+// exported value for testing
+float64 runtime·hashLoad = LOAD;