Imported Upstream version 1.4upstream/1.4

1 files changed, 0 insertions, 2892 deletions

diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
deleted file mode 100644
index 392da535b..000000000
--- a/src/pkg/runtime/mgc0.c
+++ /dev/null
@@ -1,2892 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Garbage collector (GC).
-//
-// GC is:
-// - mark&sweep
-// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
-// - parallel (up to MaxGcproc threads)
-// - partially concurrent (mark is stop-the-world, while sweep is concurrent)
-// - non-moving/non-compacting
-// - full (non-partial)
-//
-// GC rate.
-// Next GC is after we've allocated an extra amount of memory proportional to
-// the amount already in use. The proportion is controlled by GOGC environment variable
-// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
-// (this mark is tracked in next_gc variable). This keeps the GC cost in linear
-// proportion to the allocation cost. Adjusting GOGC just changes the linear constant
-// (and also the amount of extra memory used).
-//
-// Concurrent sweep.
-// The sweep phase proceeds concurrently with normal program execution.
-// The heap is swept span-by-span both lazily (when a goroutine needs another span)
-// and concurrently in a background goroutine (this helps programs that are not CPU bound).
-// However, at the end of the stop-the-world GC phase we don't know the size of the live heap,
-// and so next_gc calculation is tricky and happens as follows.
-// At the end of the stop-the-world phase next_gc is conservatively set based on total
-// heap size; all spans are marked as "needs sweeping".
-// Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory.
-// The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc
-// closer to the target value. However, this is not enough to avoid over-allocating memory.
-// Consider that a goroutine wants to allocate a new span for a large object and
-// there are no free swept spans, but there are small-object unswept spans.
-// If the goroutine naively allocates a new span, it can surpass the yet-unknown
-// target next_gc value. In order to prevent such cases (1) when a goroutine needs
-// to allocate a new small-object span, it sweeps small-object spans for the same
-// object size until it frees at least one object; (2) when a goroutine needs to
-// allocate large-object span from heap, it sweeps spans until it frees at least
-// that many pages into heap. Together these two measures ensure that we don't surpass
-// target next_gc value by a large margin. There is an exception: if a goroutine sweeps
-// and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span,
-// but there can still be other one-page unswept spans which could be combined into a two-page span.
-// It's critical to ensure that no operations proceed on unswept spans (that would corrupt
-// mark bits in GC bitmap). During GC all mcaches are flushed into the central cache,
-// so they are empty. When a goroutine grabs a new span into mcache, it sweeps it.
-// When a goroutine explicitly frees an object or sets a finalizer, it ensures that
-// the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish).
-// The finalizer goroutine is kicked off only when all spans are swept.
-// When the next GC starts, it sweeps all not-yet-swept spans (if any).
-
-#include "runtime.h"
-#include "arch_GOARCH.h"
-#include "malloc.h"
-#include "stack.h"
-#include "mgc0.h"
-#include "chan.h"
-#include "race.h"
-#include "type.h"
-#include "typekind.h"
-#include "funcdata.h"
-#include "../../cmd/ld/textflag.h"
-
-enum {
-	Debug = 0,
-	CollectStats = 0,
-	ConcurrentSweep = 1,
-
-	WorkbufSize	= 16*1024,
-	FinBlockSize	= 4*1024,
-
-	handoffThreshold = 4,
-	IntermediateBufferCapacity = 64,
-
-	// Bits in type information
-	PRECISE = 1,
-	LOOP = 2,
-	PC_BITS = PRECISE | LOOP,
-
-	RootData	= 0,
-	RootBss		= 1,
-	RootFinalizers	= 2,
-	RootSpanTypes	= 3,
-	RootFlushCaches = 4,
-	RootCount	= 5,
-};
-
-#define GcpercentUnknown (-2)
-
-// Initialized from $GOGC.  GOGC=off means no gc.
-static int32 gcpercent = GcpercentUnknown;
-
-static FuncVal* poolcleanup;
-
-void
-sync·runtime_registerPoolCleanup(FuncVal *f)
-{
-	poolcleanup = f;
-}
-
-static void
-clearpools(void)
-{
-	P *p, **pp;
-	MCache *c;
-	int32 i;
-
-	// clear sync.Pool's
-	if(poolcleanup != nil)
-		reflect·call(poolcleanup, nil, 0, 0);
-
-	for(pp=runtime·allp; p=*pp; pp++) {
-		// clear tinyalloc pool
-		c = p->mcache;
-		if(c != nil) {
-			c->tiny = nil;
-			c->tinysize = 0;
-		}
-		// clear defer pools
-		for(i=0; i<nelem(p->deferpool); i++)
-			p->deferpool[i] = nil;
-	}
-}
-
-// Holding worldsema grants an M the right to try to stop the world.
-// The procedure is:
-//
-//	runtime·semacquire(&runtime·worldsema);
-//	m->gcing = 1;
-//	runtime·stoptheworld();
-//
-//	... do stuff ...
-//
-//	m->gcing = 0;
-//	runtime·semrelease(&runtime·worldsema);
-//	runtime·starttheworld();
-//
-uint32 runtime·worldsema = 1;
-
-typedef struct Obj Obj;
-struct Obj
-{
-	byte	*p;	// data pointer
-	uintptr	n;	// size of data in bytes
-	uintptr	ti;	// type info
-};
-
-typedef struct Workbuf Workbuf;
-struct Workbuf
-{
-#define SIZE (WorkbufSize-sizeof(LFNode)-sizeof(uintptr))
-	LFNode  node; // must be first
-	uintptr nobj;
-	Obj     obj[SIZE/sizeof(Obj) - 1];
-	uint8   _padding[SIZE%sizeof(Obj) + sizeof(Obj)];
-#undef SIZE
-};
-
-typedef struct Finalizer Finalizer;
-struct Finalizer
-{
-	FuncVal *fn;
-	void *arg;
-	uintptr nret;
-	Type *fint;
-	PtrType *ot;
-};
-
-typedef struct FinBlock FinBlock;
-struct FinBlock
-{
-	FinBlock *alllink;
-	FinBlock *next;
-	int32 cnt;
-	int32 cap;
-	Finalizer fin[1];
-};
-
-extern byte data[];
-extern byte edata[];
-extern byte bss[];
-extern byte ebss[];
-
-extern byte gcdata[];
-extern byte gcbss[];
-
-static Lock	finlock;	// protects the following variables
-static FinBlock	*finq;		// list of finalizers that are to be executed
-static FinBlock	*finc;		// cache of free blocks
-static FinBlock	*allfin;	// list of all blocks
-bool	runtime·fingwait;
-bool	runtime·fingwake;
-
-static Lock	gclock;
-static G*	fing;
-
-static void	runfinq(void);
-static void	bgsweep(void);
-static Workbuf* getempty(Workbuf*);
-static Workbuf* getfull(Workbuf*);
-static void	putempty(Workbuf*);
-static Workbuf* handoff(Workbuf*);
-static void	gchelperstart(void);
-static void	flushallmcaches(void);
-static bool	scanframe(Stkframe *frame, void *wbufp);
-static void	addstackroots(G *gp, Workbuf **wbufp);
-
-static FuncVal runfinqv = {runfinq};
-static FuncVal bgsweepv = {bgsweep};
-
-static struct {
-	uint64	full;  // lock-free list of full blocks
-	uint64	empty; // lock-free list of empty blocks
-	byte	pad0[CacheLineSize]; // prevents false-sharing between full/empty and nproc/nwait
-	uint32	nproc;
-	int64	tstart;
-	volatile uint32	nwait;
-	volatile uint32	ndone;
-	Note	alldone;
-	ParFor	*markfor;
-
-	Lock;
-	byte	*chunk;
-	uintptr	nchunk;
-} work;
-
-enum {
-	GC_DEFAULT_PTR = GC_NUM_INSTR,
-	GC_CHAN,
-
-	GC_NUM_INSTR2
-};
-
-static struct {
-	struct {
-		uint64 sum;
-		uint64 cnt;
-	} ptr;
-	uint64 nbytes;
-	struct {
-		uint64 sum;
-		uint64 cnt;
-		uint64 notype;
-		uint64 typelookup;
-	} obj;
-	uint64 rescan;
-	uint64 rescanbytes;
-	uint64 instr[GC_NUM_INSTR2];
-	uint64 putempty;
-	uint64 getfull;
-	struct {
-		uint64 foundbit;
-		uint64 foundword;
-		uint64 foundspan;
-	} flushptrbuf;
-	struct {
-		uint64 foundbit;
-		uint64 foundword;
-		uint64 foundspan;
-	} markonly;
-	uint32 nbgsweep;
-	uint32 npausesweep;
-} gcstats;
-
-// markonly marks an object. It returns true if the object
-// has been marked by this function, false otherwise.
-// This function doesn't append the object to any buffer.
-static bool
-markonly(void *obj)
-{
-	byte *p;
-	uintptr *bitp, bits, shift, x, xbits, off, j;
-	MSpan *s;
-	PageID k;
-
-	// Words outside the arena cannot be pointers.
-	if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
-		return false;
-
-	// obj may be a pointer to a live object.
-	// Try to find the beginning of the object.
-
-	// Round down to word boundary.
-	obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
-
-	// Find bits for this word.
-	off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
-	bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	shift = off % wordsPerBitmapWord;
-	xbits = *bitp;
-	bits = xbits >> shift;
-
-	// Pointing at the beginning of a block?
-	if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
-		if(CollectStats)
-			runtime·xadd64(&gcstats.markonly.foundbit, 1);
-		goto found;
-	}
-
-	// Pointing just past the beginning?
-	// Scan backward a little to find a block boundary.
-	for(j=shift; j-->0; ) {
-		if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
-			shift = j;
-			bits = xbits>>shift;
-			if(CollectStats)
-				runtime·xadd64(&gcstats.markonly.foundword, 1);
-			goto found;
-		}
-	}
-
-	// Otherwise consult span table to find beginning.
-	// (Manually inlined copy of MHeap_LookupMaybe.)
-	k = (uintptr)obj>>PageShift;
-	x = k;
-	x -= (uintptr)runtime·mheap.arena_start>>PageShift;
-	s = runtime·mheap.spans[x];
-	if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
-		return false;
-	p = (byte*)((uintptr)s->start<<PageShift);
-	if(s->sizeclass == 0) {
-		obj = p;
-	} else {
-		uintptr size = s->elemsize;
-		int32 i = ((byte*)obj - p)/size;
-		obj = p+i*size;
-	}
-
-	// Now that we know the object header, reload bits.
-	off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
-	bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	shift = off % wordsPerBitmapWord;
-	xbits = *bitp;
-	bits = xbits >> shift;
-	if(CollectStats)
-		runtime·xadd64(&gcstats.markonly.foundspan, 1);
-
-found:
-	// Now we have bits, bitp, and shift correct for
-	// obj pointing at the base of the object.
-	// Only care about allocated and not marked.
-	if((bits & (bitAllocated|bitMarked)) != bitAllocated)
-		return false;
-	if(work.nproc == 1)
-		*bitp |= bitMarked<<shift;
-	else {
-		for(;;) {
-			x = *bitp;
-			if(x & (bitMarked<<shift))
-				return false;
-			if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
-				break;
-		}
-	}
-
-	// The object is now marked
-	return true;
-}
-
-// PtrTarget is a structure used by intermediate buffers.
-// The intermediate buffers hold GC data before it
-// is moved/flushed to the work buffer (Workbuf).
-// The size of an intermediate buffer is very small,
-// such as 32 or 64 elements.
-typedef struct PtrTarget PtrTarget;
-struct PtrTarget
-{
-	void *p;
-	uintptr ti;
-};
-
-typedef	struct Scanbuf Scanbuf;
-struct	Scanbuf
-{
-	struct {
-		PtrTarget *begin;
-		PtrTarget *end;
-		PtrTarget *pos;
-	} ptr;
-	struct {
-		Obj *begin;
-		Obj *end;
-		Obj *pos;
-	} obj;
-	Workbuf *wbuf;
-	Obj *wp;
-	uintptr nobj;
-};
-
-typedef struct BufferList BufferList;
-struct BufferList
-{
-	PtrTarget ptrtarget[IntermediateBufferCapacity];
-	Obj obj[IntermediateBufferCapacity];
-	uint32 busy;
-	byte pad[CacheLineSize];
-};
-#pragma dataflag NOPTR
-static BufferList bufferList[MaxGcproc];
-
-static Type *itabtype;
-
-static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
-
-// flushptrbuf moves data from the PtrTarget buffer to the work buffer.
-// The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
-// while the work buffer contains blocks which have been marked
-// and are prepared to be scanned by the garbage collector.
-//
-// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
-//
-// A simplified drawing explaining how the todo-list moves from a structure to another:
-//
-//     scanblock
-//  (find pointers)
-//    Obj ------> PtrTarget (pointer targets)
-//     ↑          |
-//     |          |
-//     `----------'
-//     flushptrbuf
-//  (find block start, mark and enqueue)
-static void
-flushptrbuf(Scanbuf *sbuf)
-{
-	byte *p, *arena_start, *obj;
-	uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n;
-	MSpan *s;
-	PageID k;
-	Obj *wp;
-	Workbuf *wbuf;
-	PtrTarget *ptrbuf;
-	PtrTarget *ptrbuf_end;
-
-	arena_start = runtime·mheap.arena_start;
-
-	wp = sbuf->wp;
-	wbuf = sbuf->wbuf;
-	nobj = sbuf->nobj;
-
-	ptrbuf = sbuf->ptr.begin;
-	ptrbuf_end = sbuf->ptr.pos;
-	n = ptrbuf_end - sbuf->ptr.begin;
-	sbuf->ptr.pos = sbuf->ptr.begin;
-
-	if(CollectStats) {
-		runtime·xadd64(&gcstats.ptr.sum, n);
-		runtime·xadd64(&gcstats.ptr.cnt, 1);
-	}
-
-	// If buffer is nearly full, get a new one.
-	if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) {
-		if(wbuf != nil)
-			wbuf->nobj = nobj;
-		wbuf = getempty(wbuf);
-		wp = wbuf->obj;
-		nobj = 0;
-
-		if(n >= nelem(wbuf->obj))
-			runtime·throw("ptrbuf has to be smaller than WorkBuf");
-	}
-
-	while(ptrbuf < ptrbuf_end) {
-		obj = ptrbuf->p;
-		ti = ptrbuf->ti;
-		ptrbuf++;
-
-		// obj belongs to interval [mheap.arena_start, mheap.arena_used).
-		if(Debug > 1) {
-			if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
-				runtime·throw("object is outside of mheap");
-		}
-
-		// obj may be a pointer to a live object.
-		// Try to find the beginning of the object.
-
-		// Round down to word boundary.
-		if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) {
-			obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
-			ti = 0;
-		}
-
-		// Find bits for this word.
-		off = (uintptr*)obj - (uintptr*)arena_start;
-		bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		xbits = *bitp;
-		bits = xbits >> shift;
-
-		// Pointing at the beginning of a block?
-		if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
-			if(CollectStats)
-				runtime·xadd64(&gcstats.flushptrbuf.foundbit, 1);
-			goto found;
-		}
-
-		ti = 0;
-
-		// Pointing just past the beginning?
-		// Scan backward a little to find a block boundary.
-		for(j=shift; j-->0; ) {
-			if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
-				obj = (byte*)obj - (shift-j)*PtrSize;
-				shift = j;
-				bits = xbits>>shift;
-				if(CollectStats)
-					runtime·xadd64(&gcstats.flushptrbuf.foundword, 1);
-				goto found;
-			}
-		}
-
-		// Otherwise consult span table to find beginning.
-		// (Manually inlined copy of MHeap_LookupMaybe.)
-		k = (uintptr)obj>>PageShift;
-		x = k;
-		x -= (uintptr)arena_start>>PageShift;
-		s = runtime·mheap.spans[x];
-		if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
-			continue;
-		p = (byte*)((uintptr)s->start<<PageShift);
-		if(s->sizeclass == 0) {
-			obj = p;
-		} else {
-			size = s->elemsize;
-			int32 i = ((byte*)obj - p)/size;
-			obj = p+i*size;
-		}
-
-		// Now that we know the object header, reload bits.
-		off = (uintptr*)obj - (uintptr*)arena_start;
-		bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		xbits = *bitp;
-		bits = xbits >> shift;
-		if(CollectStats)
-			runtime·xadd64(&gcstats.flushptrbuf.foundspan, 1);
-
-	found:
-		// Now we have bits, bitp, and shift correct for
-		// obj pointing at the base of the object.
-		// Only care about allocated and not marked.
-		if((bits & (bitAllocated|bitMarked)) != bitAllocated)
-			continue;
-		if(work.nproc == 1)
-			*bitp |= bitMarked<<shift;
-		else {
-			for(;;) {
-				x = *bitp;
-				if(x & (bitMarked<<shift))
-					goto continue_obj;
-				if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
-					break;
-			}
-		}
-
-		// If object has no pointers, don't need to scan further.
-		if((bits & bitScan) == 0)
-			continue;
-
-		// Ask span about size class.
-		// (Manually inlined copy of MHeap_Lookup.)
-		x = (uintptr)obj >> PageShift;
-		x -= (uintptr)arena_start>>PageShift;
-		s = runtime·mheap.spans[x];
-
-		PREFETCH(obj);
-
-		*wp = (Obj){obj, s->elemsize, ti};
-		wp++;
-		nobj++;
-	continue_obj:;
-	}
-
-	// If another proc wants a pointer, give it some.
-	if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
-		wbuf->nobj = nobj;
-		wbuf = handoff(wbuf);
-		nobj = wbuf->nobj;
-		wp = wbuf->obj + nobj;
-	}
-
-	sbuf->wp = wp;
-	sbuf->wbuf = wbuf;
-	sbuf->nobj = nobj;
-}
-
-static void
-flushobjbuf(Scanbuf *sbuf)
-{
-	uintptr nobj, off;
-	Obj *wp, obj;
-	Workbuf *wbuf;
-	Obj *objbuf;
-	Obj *objbuf_end;
-
-	wp = sbuf->wp;
-	wbuf = sbuf->wbuf;
-	nobj = sbuf->nobj;
-
-	objbuf = sbuf->obj.begin;
-	objbuf_end = sbuf->obj.pos;
-	sbuf->obj.pos = sbuf->obj.begin;
-
-	while(objbuf < objbuf_end) {
-		obj = *objbuf++;
-
-		// Align obj.b to a word boundary.
-		off = (uintptr)obj.p & (PtrSize-1);
-		if(off != 0) {
-			obj.p += PtrSize - off;
-			obj.n -= PtrSize - off;
-			obj.ti = 0;
-		}
-
-		if(obj.p == nil || obj.n == 0)
-			continue;
-
-		// If buffer is full, get a new one.
-		if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
-			if(wbuf != nil)
-				wbuf->nobj = nobj;
-			wbuf = getempty(wbuf);
-			wp = wbuf->obj;
-			nobj = 0;
-		}
-
-		*wp = obj;
-		wp++;
-		nobj++;
-	}
-
-	// If another proc wants a pointer, give it some.
-	if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
-		wbuf->nobj = nobj;
-		wbuf = handoff(wbuf);
-		nobj = wbuf->nobj;
-		wp = wbuf->obj + nobj;
-	}
-
-	sbuf->wp = wp;
-	sbuf->wbuf = wbuf;
-	sbuf->nobj = nobj;
-}
-
-// Program that scans the whole block and treats every block element as a potential pointer
-static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
-
-// Hchan program
-static uintptr chanProg[2] = {0, GC_CHAN};
-
-// Local variables of a program fragment or loop
-typedef struct Frame Frame;
-struct Frame {
-	uintptr count, elemsize, b;
-	uintptr *loop_or_ret;
-};
-
-// Sanity check for the derived type info objti.
-static void
-checkptr(void *obj, uintptr objti)
-{
-	uintptr *pc1, *pc2, type, tisize, i, j, x;
-	byte *objstart;
-	Type *t;
-	MSpan *s;
-
-	if(!Debug)
-		runtime·throw("checkptr is debug only");
-
-	if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
-		return;
-	type = runtime·gettype(obj);
-	t = (Type*)(type & ~(uintptr)(PtrSize-1));
-	if(t == nil)
-		return;
-	x = (uintptr)obj >> PageShift;
-	x -= (uintptr)(runtime·mheap.arena_start)>>PageShift;
-	s = runtime·mheap.spans[x];
-	objstart = (byte*)((uintptr)s->start<<PageShift);
-	if(s->sizeclass != 0) {
-		i = ((byte*)obj - objstart)/s->elemsize;
-		objstart += i*s->elemsize;
-	}
-	tisize = *(uintptr*)objti;
-	// Sanity check for object size: it should fit into the memory block.
-	if((byte*)obj + tisize > objstart + s->elemsize) {
-		runtime·printf("object of type '%S' at %p/%p does not fit in block %p/%p\n",
-			       *t->string, obj, tisize, objstart, s->elemsize);
-		runtime·throw("invalid gc type info");
-	}
-	if(obj != objstart)
-		return;
-	// If obj points to the beginning of the memory block,
-	// check type info as well.
-	if(t->string == nil ||
-		// Gob allocates unsafe pointers for indirection.
-		(runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") &&
-		// Runtime and gc think differently about closures.
-		runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t->string->str)) {
-		pc1 = (uintptr*)objti;
-		pc2 = (uintptr*)t->gc;
-		// A simple best-effort check until first GC_END.
-		for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) {
-			if(pc1[j] != pc2[j]) {
-				runtime·printf("invalid gc type info for '%s', type info %p [%d]=%p, block info %p [%d]=%p\n",
-					       t->string ? (int8*)t->string->str : (int8*)"?", pc1, (int32)j, pc1[j], pc2, (int32)j, pc2[j]);
-				runtime·throw("invalid gc type info");
-			}
-		}
-	}
-}					
-
-// scanblock scans a block of n bytes starting at pointer b for references
-// to other objects, scanning any it finds recursively until there are no
-// unscanned objects left.  Instead of using an explicit recursion, it keeps
-// a work list in the Workbuf* structures and loops in the main function
-// body.  Keeping an explicit work list is easier on the stack allocator and
-// more efficient.
-static void
-scanblock(Workbuf *wbuf, bool keepworking)
-{
-	byte *b, *arena_start, *arena_used;
-	uintptr n, i, end_b, elemsize, size, ti, objti, count, type, nobj;
-	uintptr *pc, precise_type, nominal_size;
-	uintptr *chan_ret, chancap;
-	void *obj;
-	Type *t, *et;
-	Slice *sliceptr;
-	String *stringptr;
-	Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
-	BufferList *scanbuffers;
-	Scanbuf sbuf;
-	Eface *eface;
-	Iface *iface;
-	Hchan *chan;
-	ChanType *chantype;
-	Obj *wp;
-
-	if(sizeof(Workbuf) % WorkbufSize != 0)
-		runtime·throw("scanblock: size of Workbuf is suboptimal");
-
-	// Memory arena parameters.
-	arena_start = runtime·mheap.arena_start;
-	arena_used = runtime·mheap.arena_used;
-
-	stack_ptr = stack+nelem(stack)-1;
-
-	precise_type = false;
-	nominal_size = 0;
-
-	if(wbuf) {
-		nobj = wbuf->nobj;
-		wp = &wbuf->obj[nobj];
-	} else {
-		nobj = 0;
-		wp = nil;
-	}
-
-	// Initialize sbuf
-	scanbuffers = &bufferList[m->helpgc];
-
-	sbuf.ptr.begin = sbuf.ptr.pos = &scanbuffers->ptrtarget[0];
-	sbuf.ptr.end = sbuf.ptr.begin + nelem(scanbuffers->ptrtarget);
-
-	sbuf.obj.begin = sbuf.obj.pos = &scanbuffers->obj[0];
-	sbuf.obj.end = sbuf.obj.begin + nelem(scanbuffers->obj);
-
-	sbuf.wbuf = wbuf;
-	sbuf.wp = wp;
-	sbuf.nobj = nobj;
-
-	// (Silence the compiler)
-	chan = nil;
-	chantype = nil;
-	chan_ret = nil;
-
-	goto next_block;
-
-	for(;;) {
-		// Each iteration scans the block b of length n, queueing pointers in
-		// the work buffer.
-
-		if(CollectStats) {
-			runtime·xadd64(&gcstats.nbytes, n);
-			runtime·xadd64(&gcstats.obj.sum, sbuf.nobj);
-			runtime·xadd64(&gcstats.obj.cnt, 1);
-		}
-
-		if(ti != 0) {
-			if(Debug > 1) {
-				runtime·printf("scanblock %p %D ti %p\n", b, (int64)n, ti);
-			}
-			pc = (uintptr*)(ti & ~(uintptr)PC_BITS);
-			precise_type = (ti & PRECISE);
-			stack_top.elemsize = pc[0];
-			if(!precise_type)
-				nominal_size = pc[0];
-			if(ti & LOOP) {
-				stack_top.count = 0;	// 0 means an infinite number of iterations
-				stack_top.loop_or_ret = pc+1;
-			} else {
-				stack_top.count = 1;
-			}
-			if(Debug) {
-				// Simple sanity check for provided type info ti:
-				// The declared size of the object must be not larger than the actual size
-				// (it can be smaller due to inferior pointers).
-				// It's difficult to make a comprehensive check due to inferior pointers,
-				// reflection, gob, etc.
-				if(pc[0] > n) {
-					runtime·printf("invalid gc type info: type info size %p, block size %p\n", pc[0], n);
-					runtime·throw("invalid gc type info");
-				}
-			}
-		} else if(UseSpanType) {
-			if(CollectStats)
-				runtime·xadd64(&gcstats.obj.notype, 1);
-
-			type = runtime·gettype(b);
-			if(type != 0) {
-				if(CollectStats)
-					runtime·xadd64(&gcstats.obj.typelookup, 1);
-
-				t = (Type*)(type & ~(uintptr)(PtrSize-1));
-				switch(type & (PtrSize-1)) {
-				case TypeInfo_SingleObject:
-					pc = (uintptr*)t->gc;
-					precise_type = true;  // type information about 'b' is precise
-					stack_top.count = 1;
-					stack_top.elemsize = pc[0];
-					break;
-				case TypeInfo_Array:
-					pc = (uintptr*)t->gc;
-					if(pc[0] == 0)
-						goto next_block;
-					precise_type = true;  // type information about 'b' is precise
-					stack_top.count = 0;  // 0 means an infinite number of iterations
-					stack_top.elemsize = pc[0];
-					stack_top.loop_or_ret = pc+1;
-					break;
-				case TypeInfo_Chan:
-					chan = (Hchan*)b;
-					chantype = (ChanType*)t;
-					chan_ret = nil;
-					pc = chanProg;
-					break;
-				default:
-					if(Debug > 1)
-						runtime·printf("scanblock %p %D type %p %S\n", b, (int64)n, type, *t->string);
-					runtime·throw("scanblock: invalid type");
-					return;
-				}
-				if(Debug > 1)
-					runtime·printf("scanblock %p %D type %p %S pc=%p\n", b, (int64)n, type, *t->string, pc);
-			} else {
-				pc = defaultProg;
-				if(Debug > 1)
-					runtime·printf("scanblock %p %D unknown type\n", b, (int64)n);
-			}
-		} else {
-			pc = defaultProg;
-			if(Debug > 1)
-				runtime·printf("scanblock %p %D no span types\n", b, (int64)n);
-		}
-
-		if(IgnorePreciseGC)
-			pc = defaultProg;
-
-		pc++;
-		stack_top.b = (uintptr)b;
-		end_b = (uintptr)b + n - PtrSize;
-
-	for(;;) {
-		if(CollectStats)
-			runtime·xadd64(&gcstats.instr[pc[0]], 1);
-
-		obj = nil;
-		objti = 0;
-		switch(pc[0]) {
-		case GC_PTR:
-			obj = *(void**)(stack_top.b + pc[1]);
-			objti = pc[2];
-			if(Debug > 2)
-				runtime·printf("gc_ptr @%p: %p ti=%p\n", stack_top.b+pc[1], obj, objti);
-			pc += 3;
-			if(Debug)
-				checkptr(obj, objti);
-			break;
-
-		case GC_SLICE:
-			sliceptr = (Slice*)(stack_top.b + pc[1]);
-			if(Debug > 2)
-				runtime·printf("gc_slice @%p: %p/%D/%D\n", sliceptr, sliceptr->array, (int64)sliceptr->len, (int64)sliceptr->cap);
-			if(sliceptr->cap != 0) {
-				obj = sliceptr->array;
-				// Can't use slice element type for scanning,
-				// because if it points to an array embedded
-				// in the beginning of a struct,
-				// we will scan the whole struct as the slice.
-				// So just obtain type info from heap.
-			}
-			pc += 3;
-			break;
-
-		case GC_APTR:
-			obj = *(void**)(stack_top.b + pc[1]);
-			if(Debug > 2)
-				runtime·printf("gc_aptr @%p: %p\n", stack_top.b+pc[1], obj);
-			pc += 2;
-			break;
-
-		case GC_STRING:
-			stringptr = (String*)(stack_top.b + pc[1]);
-			if(Debug > 2)
-				runtime·printf("gc_string @%p: %p/%D\n", stack_top.b+pc[1], stringptr->str, (int64)stringptr->len);
-			if(stringptr->len != 0)
-				markonly(stringptr->str);
-			pc += 2;
-			continue;
-
-		case GC_EFACE:
-			eface = (Eface*)(stack_top.b + pc[1]);
-			pc += 2;
-			if(Debug > 2)
-				runtime·printf("gc_eface @%p: %p %p\n", stack_top.b+pc[1], eface->type, eface->data);
-			if(eface->type == nil)
-				continue;
-
-			// eface->type
-			t = eface->type;
-			if((void*)t >= arena_start && (void*)t < arena_used) {
-				*sbuf.ptr.pos++ = (PtrTarget){t, 0};
-				if(sbuf.ptr.pos == sbuf.ptr.end)
-					flushptrbuf(&sbuf);
-			}
-
-			// eface->data
-			if(eface->data >= arena_start && eface->data < arena_used) {
-				if(t->size <= sizeof(void*)) {
-					if((t->kind & KindNoPointers))
-						continue;
-
-					obj = eface->data;
-					if((t->kind & ~KindNoPointers) == KindPtr) {
-						// Only use type information if it is a pointer-containing type.
-						// This matches the GC programs written by cmd/gc/reflect.c's
-						// dgcsym1 in case TPTR32/case TPTR64. See rationale there.
-						et = ((PtrType*)t)->elem;
-						if(!(et->kind & KindNoPointers))
-							objti = (uintptr)((PtrType*)t)->elem->gc;
-					}
-				} else {
-					obj = eface->data;
-					objti = (uintptr)t->gc;
-				}
-			}
-			break;
-
-		case GC_IFACE:
-			iface = (Iface*)(stack_top.b + pc[1]);
-			pc += 2;
-			if(Debug > 2)
-				runtime·printf("gc_iface @%p: %p/%p %p\n", stack_top.b+pc[1], iface->tab, nil, iface->data);
-			if(iface->tab == nil)
-				continue;
-			
-			// iface->tab
-			if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) {
-				*sbuf.ptr.pos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc};
-				if(sbuf.ptr.pos == sbuf.ptr.end)
-					flushptrbuf(&sbuf);
-			}
-
-			// iface->data
-			if(iface->data >= arena_start && iface->data < arena_used) {
-				t = iface->tab->type;
-				if(t->size <= sizeof(void*)) {
-					if((t->kind & KindNoPointers))
-						continue;
-
-					obj = iface->data;
-					if((t->kind & ~KindNoPointers) == KindPtr) {
-						// Only use type information if it is a pointer-containing type.
-						// This matches the GC programs written by cmd/gc/reflect.c's
-						// dgcsym1 in case TPTR32/case TPTR64. See rationale there.
-						et = ((PtrType*)t)->elem;
-						if(!(et->kind & KindNoPointers))
-							objti = (uintptr)((PtrType*)t)->elem->gc;
-					}
-				} else {
-					obj = iface->data;
-					objti = (uintptr)t->gc;
-				}
-			}
-			break;
-
-		case GC_DEFAULT_PTR:
-			while(stack_top.b <= end_b) {
-				obj = *(byte**)stack_top.b;
-				if(Debug > 2)
-					runtime·printf("gc_default_ptr @%p: %p\n", stack_top.b, obj);
-				stack_top.b += PtrSize;
-				if(obj >= arena_start && obj < arena_used) {
-					*sbuf.ptr.pos++ = (PtrTarget){obj, 0};
-					if(sbuf.ptr.pos == sbuf.ptr.end)
-						flushptrbuf(&sbuf);
-				}
-			}
-			goto next_block;
-
-		case GC_END:
-			if(--stack_top.count != 0) {
-				// Next iteration of a loop if possible.
-				stack_top.b += stack_top.elemsize;
-				if(stack_top.b + stack_top.elemsize <= end_b+PtrSize) {
-					pc = stack_top.loop_or_ret;
-					continue;
-				}
-				i = stack_top.b;
-			} else {
-				// Stack pop if possible.
-				if(stack_ptr+1 < stack+nelem(stack)) {
-					pc = stack_top.loop_or_ret;
-					stack_top = *(++stack_ptr);
-					continue;
-				}
-				i = (uintptr)b + nominal_size;
-			}
-			if(!precise_type) {
-				// Quickly scan [b+i,b+n) for possible pointers.
-				for(; i<=end_b; i+=PtrSize) {
-					if(*(byte**)i != nil) {
-						// Found a value that may be a pointer.
-						// Do a rescan of the entire block.
-						enqueue((Obj){b, n, 0}, &sbuf.wbuf, &sbuf.wp, &sbuf.nobj);
-						if(CollectStats) {
-							runtime·xadd64(&gcstats.rescan, 1);
-							runtime·xadd64(&gcstats.rescanbytes, n);
-						}
-						break;
-					}
-				}
-			}
-			goto next_block;
-
-		case GC_ARRAY_START:
-			i = stack_top.b + pc[1];
-			count = pc[2];
-			elemsize = pc[3];
-			pc += 4;
-
-			// Stack push.
-			*stack_ptr-- = stack_top;
-			stack_top = (Frame){count, elemsize, i, pc};
-			continue;
-
-		case GC_ARRAY_NEXT:
-			if(--stack_top.count != 0) {
-				stack_top.b += stack_top.elemsize;
-				pc = stack_top.loop_or_ret;
-			} else {
-				// Stack pop.
-				stack_top = *(++stack_ptr);
-				pc += 1;
-			}
-			continue;
-
-		case GC_CALL:
-			// Stack push.
-			*stack_ptr-- = stack_top;
-			stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*return address*/};
-			pc = (uintptr*)((byte*)pc + *(int32*)(pc+2));  // target of the CALL instruction
-			continue;
-
-		case GC_REGION:
-			obj = (void*)(stack_top.b + pc[1]);
-			size = pc[2];
-			objti = pc[3];
-			pc += 4;
-
-			if(Debug > 2)
-				runtime·printf("gc_region @%p: %D %p\n", stack_top.b+pc[1], (int64)size, objti);
-			*sbuf.obj.pos++ = (Obj){obj, size, objti};
-			if(sbuf.obj.pos == sbuf.obj.end)
-				flushobjbuf(&sbuf);
-			continue;
-
-		case GC_CHAN_PTR:
-			chan = *(Hchan**)(stack_top.b + pc[1]);
-			if(Debug > 2 && chan != nil)
-				runtime·printf("gc_chan_ptr @%p: %p/%D/%D %p\n", stack_top.b+pc[1], chan, (int64)chan->qcount, (int64)chan->dataqsiz, pc[2]);
-			if(chan == nil) {
-				pc += 3;
-				continue;
-			}
-			if(markonly(chan)) {
-				chantype = (ChanType*)pc[2];
-				if(!(chantype->elem->kind & KindNoPointers)) {
-					// Start chanProg.
-					chan_ret = pc+3;
-					pc = chanProg+1;
-					continue;
-				}
-			}
-			pc += 3;
-			continue;
-
-		case GC_CHAN:
-			// There are no heap pointers in struct Hchan,
-			// so we can ignore the leading sizeof(Hchan) bytes.
-			if(!(chantype->elem->kind & KindNoPointers)) {
-				// Channel's buffer follows Hchan immediately in memory.
-				// Size of buffer (cap(c)) is second int in the chan struct.
-				chancap = ((uintgo*)chan)[1];
-				if(chancap > 0) {
-					// TODO(atom): split into two chunks so that only the
-					// in-use part of the circular buffer is scanned.
-					// (Channel routines zero the unused part, so the current
-					// code does not lead to leaks, it's just a little inefficient.)
-					*sbuf.obj.pos++ = (Obj){(byte*)chan+runtime·Hchansize, chancap*chantype->elem->size,
-						(uintptr)chantype->elem->gc | PRECISE | LOOP};
-					if(sbuf.obj.pos == sbuf.obj.end)
-						flushobjbuf(&sbuf);
-				}
-			}
-			if(chan_ret == nil)
-				goto next_block;
-			pc = chan_ret;
-			continue;
-
-		default:
-			runtime·printf("runtime: invalid GC instruction %p at %p\n", pc[0], pc);
-			runtime·throw("scanblock: invalid GC instruction");
-			return;
-		}
-
-		if(obj >= arena_start && obj < arena_used) {
-			*sbuf.ptr.pos++ = (PtrTarget){obj, objti};
-			if(sbuf.ptr.pos == sbuf.ptr.end)
-				flushptrbuf(&sbuf);
-		}
-	}
-
-	next_block:
-		// Done scanning [b, b+n).  Prepare for the next iteration of
-		// the loop by setting b, n, ti to the parameters for the next block.
-
-		if(sbuf.nobj == 0) {
-			flushptrbuf(&sbuf);
-			flushobjbuf(&sbuf);
-
-			if(sbuf.nobj == 0) {
-				if(!keepworking) {
-					if(sbuf.wbuf)
-						putempty(sbuf.wbuf);
-					return;
-				}
-				// Emptied our buffer: refill.
-				sbuf.wbuf = getfull(sbuf.wbuf);
-				if(sbuf.wbuf == nil)
-					return;
-				sbuf.nobj = sbuf.wbuf->nobj;
-				sbuf.wp = sbuf.wbuf->obj + sbuf.wbuf->nobj;
-			}
-		}
-
-		// Fetch b from the work buffer.
-		--sbuf.wp;
-		b = sbuf.wp->p;
-		n = sbuf.wp->n;
-		ti = sbuf.wp->ti;
-		sbuf.nobj--;
-	}
-}
-
-// Append obj to the work buffer.
-// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
-static void
-enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj)
-{
-	uintptr nobj, off;
-	Obj *wp;
-	Workbuf *wbuf;
-
-	if(Debug > 1)
-		runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, obj.ti);
-
-	// Align obj.b to a word boundary.
-	off = (uintptr)obj.p & (PtrSize-1);
-	if(off != 0) {
-		obj.p += PtrSize - off;
-		obj.n -= PtrSize - off;
-		obj.ti = 0;
-	}
-
-	if(obj.p == nil || obj.n == 0)
-		return;
-
-	// Load work buffer state
-	wp = *_wp;
-	wbuf = *_wbuf;
-	nobj = *_nobj;
-
-	// If another proc wants a pointer, give it some.
-	if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
-		wbuf->nobj = nobj;
-		wbuf = handoff(wbuf);
-		nobj = wbuf->nobj;
-		wp = wbuf->obj + nobj;
-	}
-
-	// If buffer is full, get a new one.
-	if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
-		if(wbuf != nil)
-			wbuf->nobj = nobj;
-		wbuf = getempty(wbuf);
-		wp = wbuf->obj;
-		nobj = 0;
-	}
-
-	*wp = obj;
-	wp++;
-	nobj++;
-
-	// Save work buffer state
-	*_wp = wp;
-	*_wbuf = wbuf;
-	*_nobj = nobj;
-}
-
-static void
-enqueue1(Workbuf **wbufp, Obj obj)
-{
-	Workbuf *wbuf;
-
-	wbuf = *wbufp;
-	if(wbuf->nobj >= nelem(wbuf->obj))
-		*wbufp = wbuf = getempty(wbuf);
-	wbuf->obj[wbuf->nobj++] = obj;
-}
-
-static void
-markroot(ParFor *desc, uint32 i)
-{
-	Workbuf *wbuf;
-	FinBlock *fb;
-	MHeap *h;
-	MSpan **allspans, *s;
-	uint32 spanidx, sg;
-	G *gp;
-	void *p;
-
-	USED(&desc);
-	wbuf = getempty(nil);
-	// Note: if you add a case here, please also update heapdump.c:dumproots.
-	switch(i) {
-	case RootData:
-		enqueue1(&wbuf, (Obj){data, edata - data, (uintptr)gcdata});
-		break;
-
-	case RootBss:
-		enqueue1(&wbuf, (Obj){bss, ebss - bss, (uintptr)gcbss});
-		break;
-
-	case RootFinalizers:
-		for(fb=allfin; fb; fb=fb->alllink)
-			enqueue1(&wbuf, (Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
-		break;
-
-	case RootSpanTypes:
-		// mark span types and MSpan.specials (to walk spans only once)
-		h = &runtime·mheap;
-		sg = h->sweepgen;
-		allspans = h->allspans;
-		for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
-			Special *sp;
-			SpecialFinalizer *spf;
-
-			s = allspans[spanidx];
-			if(s->sweepgen != sg) {
-				runtime·printf("sweep %d %d\n", s->sweepgen, sg);
-				runtime·throw("gc: unswept span");
-			}
-			if(s->state != MSpanInUse)
-				continue;
-			// The garbage collector ignores type pointers stored in MSpan.types:
-			//  - Compiler-generated types are stored outside of heap.
-			//  - The reflect package has runtime-generated types cached in its data structures.
-			//    The garbage collector relies on finding the references via that cache.
-			if(s->types.compression == MTypes_Words || s->types.compression == MTypes_Bytes)
-				markonly((byte*)s->types.data);
-			for(sp = s->specials; sp != nil; sp = sp->next) {
-				if(sp->kind != KindSpecialFinalizer)
-					continue;
-				// don't mark finalized object, but scan it so we
-				// retain everything it points to.
-				spf = (SpecialFinalizer*)sp;
-				// A finalizer can be set for an inner byte of an object, find object beginning.
-				p = (void*)((s->start << PageShift) + spf->offset/s->elemsize*s->elemsize);
-				enqueue1(&wbuf, (Obj){p, s->elemsize, 0});
-				enqueue1(&wbuf, (Obj){(void*)&spf->fn, PtrSize, 0});
-				enqueue1(&wbuf, (Obj){(void*)&spf->fint, PtrSize, 0});
-				enqueue1(&wbuf, (Obj){(void*)&spf->ot, PtrSize, 0});
-			}
-		}
-		break;
-
-	case RootFlushCaches:
-		flushallmcaches();
-		break;
-
-	default:
-		// the rest is scanning goroutine stacks
-		if(i - RootCount >= runtime·allglen)
-			runtime·throw("markroot: bad index");
-		gp = runtime·allg[i - RootCount];
-		// remember when we've first observed the G blocked
-		// needed only to output in traceback
-		if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince == 0)
-			gp->waitsince = work.tstart;
-		addstackroots(gp, &wbuf);
-		break;
-		
-	}
-
-	if(wbuf)
-		scanblock(wbuf, false);
-}
-
-// Get an empty work buffer off the work.empty list,
-// allocating new buffers as needed.
-static Workbuf*
-getempty(Workbuf *b)
-{
-	if(b != nil)
-		runtime·lfstackpush(&work.full, &b->node);
-	b = (Workbuf*)runtime·lfstackpop(&work.empty);
-	if(b == nil) {
-		// Need to allocate.
-		runtime·lock(&work);
-		if(work.nchunk < sizeof *b) {
-			work.nchunk = 1<<20;
-			work.chunk = runtime·SysAlloc(work.nchunk, &mstats.gc_sys);
-			if(work.chunk == nil)
-				runtime·throw("runtime: cannot allocate memory");
-		}
-		b = (Workbuf*)work.chunk;
-		work.chunk += sizeof *b;
-		work.nchunk -= sizeof *b;
-		runtime·unlock(&work);
-	}
-	b->nobj = 0;
-	return b;
-}
-
-static void
-putempty(Workbuf *b)
-{
-	if(CollectStats)
-		runtime·xadd64(&gcstats.putempty, 1);
-
-	runtime·lfstackpush(&work.empty, &b->node);
-}
-
-// Get a full work buffer off the work.full list, or return nil.
-static Workbuf*
-getfull(Workbuf *b)
-{
-	int32 i;
-
-	if(CollectStats)
-		runtime·xadd64(&gcstats.getfull, 1);
-
-	if(b != nil)
-		runtime·lfstackpush(&work.empty, &b->node);
-	b = (Workbuf*)runtime·lfstackpop(&work.full);
-	if(b != nil || work.nproc == 1)
-		return b;
-
-	runtime·xadd(&work.nwait, +1);
-	for(i=0;; i++) {
-		if(work.full != 0) {
-			runtime·xadd(&work.nwait, -1);
-			b = (Workbuf*)runtime·lfstackpop(&work.full);
-			if(b != nil)
-				return b;
-			runtime·xadd(&work.nwait, +1);
-		}
-		if(work.nwait == work.nproc)
-			return nil;
-		if(i < 10) {
-			m->gcstats.nprocyield++;
-			runtime·procyield(20);
-		} else if(i < 20) {
-			m->gcstats.nosyield++;
-			runtime·osyield();
-		} else {
-			m->gcstats.nsleep++;
-			runtime·usleep(100);
-		}
-	}
-}
-
-static Workbuf*
-handoff(Workbuf *b)
-{
-	int32 n;
-	Workbuf *b1;
-
-	// Make new buffer with half of b's pointers.
-	b1 = getempty(nil);
-	n = b->nobj/2;
-	b->nobj -= n;
-	b1->nobj = n;
-	runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]);
-	m->gcstats.nhandoff++;
-	m->gcstats.nhandoffcnt += n;
-
-	// Put b on full list - let first half of b get stolen.
-	runtime·lfstackpush(&work.full, &b->node);
-	return b1;
-}
-
-extern byte pclntab[]; // base for f->ptrsoff
-
-BitVector
-runtime·stackmapdata(StackMap *stackmap, int32 n)
-{
-	if(n < 0 || n >= stackmap->n)
-		runtime·throw("stackmapdata: index out of range");
-	return (BitVector){stackmap->nbit, stackmap->data + n*((stackmap->nbit+31)/32)};
-}
-
-// Scans an interface data value when the interface type indicates
-// that it is a pointer.
-static void
-scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, void *wbufp)
-{
-	Itab *tab;
-	Type *type;
-
-	if(runtime·precisestack && afterprologue) {
-		if(bits == BitsIface) {
-			tab = *(Itab**)scanp;
-			if(tab->type->size <= sizeof(void*) && (tab->type->kind & KindNoPointers))
-				return;
-		} else { // bits == BitsEface
-			type = *(Type**)scanp;
-			if(type->size <= sizeof(void*) && (type->kind & KindNoPointers))
-				return;
-		}
-	}
-	enqueue1(wbufp, (Obj){scanp+PtrSize, PtrSize, 0});
-}
-
-// Starting from scanp, scans words corresponding to set bits.
-static void
-scanbitvector(Func *f, bool precise, byte *scanp, BitVector *bv, bool afterprologue, void *wbufp)
-{
-	uintptr word, bits;
-	uint32 *wordp;
-	int32 i, remptrs;
-	byte *p;
-
-	wordp = bv->data;
-	for(remptrs = bv->n; remptrs > 0; remptrs -= 32) {
-		word = *wordp++;
-		if(remptrs < 32)
-			i = remptrs;
-		else
-			i = 32;
-		i /= BitsPerPointer;
-		for(; i > 0; i--) {
-			bits = word & 3;
-			switch(bits) {
-			case BitsDead:
-				if(runtime·debug.gcdead)
-					*(uintptr*)scanp = PoisonGC;
-				break;
-			case BitsScalar:
-				break;
-			case BitsPointer:
-				p = *(byte**)scanp;
-				if(p != nil) {
-					if(Debug > 2)
-						runtime·printf("frame %s @%p: ptr %p\n", runtime·funcname(f), scanp, p);
-					if(precise && (p < (byte*)PageSize || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) {
-						// Looks like a junk value in a pointer slot.
-						// Liveness analysis wrong?
-						m->traceback = 2;
-						runtime·printf("bad pointer in frame %s at %p: %p\n", runtime·funcname(f), scanp, p);
-						runtime·throw("bad pointer in scanbitvector");
-					}
-					enqueue1(wbufp, (Obj){scanp, PtrSize, 0});
-				}
-				break;
-			case BitsMultiWord:
-				p = scanp;
-				word >>= BitsPerPointer;
-				scanp += PtrSize;
-				i--;
-				if(i == 0) {
-					// Get next chunk of bits
-					remptrs -= 32;
-					word = *wordp++;
-					if(remptrs < 32)
-						i = remptrs;
-					else
-						i = 32;
-					i /= BitsPerPointer;
-				}
-				switch(word & 3) {
-				case BitsString:
-					if(Debug > 2)
-						runtime·printf("frame %s @%p: string %p/%D\n", runtime·funcname(f), p, ((String*)p)->str, (int64)((String*)p)->len);
-					if(((String*)p)->len != 0)
-						markonly(((String*)p)->str);
-					break;
-				case BitsSlice:
-					word >>= BitsPerPointer;
-					scanp += PtrSize;
-					i--;
-					if(i == 0) {
-						// Get next chunk of bits
-						remptrs -= 32;
-						word = *wordp++;
-						if(remptrs < 32)
-							i = remptrs;
-						else
-							i = 32;
-						i /= BitsPerPointer;
-					}
-					if(Debug > 2)
-						runtime·printf("frame %s @%p: slice %p/%D/%D\n", runtime·funcname(f), p, ((Slice*)p)->array, (int64)((Slice*)p)->len, (int64)((Slice*)p)->cap);
-					if(((Slice*)p)->cap < ((Slice*)p)->len) {
-						m->traceback = 2;
-						runtime·printf("bad slice in frame %s at %p: %p/%p/%p\n", runtime·funcname(f), p, ((byte**)p)[0], ((byte**)p)[1], ((byte**)p)[2]);
-						runtime·throw("slice capacity smaller than length");
-					}
-					if(((Slice*)p)->cap != 0)
-						enqueue1(wbufp, (Obj){p, PtrSize, 0});
-					break;
-				case BitsIface:
-				case BitsEface:
-					if(*(byte**)p != nil) {
-						if(Debug > 2) {
-							if((word&3) == BitsEface)
-								runtime·printf("frame %s @%p: eface %p %p\n", runtime·funcname(f), p, ((uintptr*)p)[0], ((uintptr*)p)[1]);
-							else
-								runtime·printf("frame %s @%p: iface %p %p\n", runtime·funcname(f), p, ((uintptr*)p)[0], ((uintptr*)p)[1]);
-						}
-						scaninterfacedata(word & 3, p, afterprologue, wbufp);
-					}
-					break;
-				}
-			}
-			word >>= BitsPerPointer;
-			scanp += PtrSize;
-		}
-	}
-}
-
-// Scan a stack frame: local variables and function arguments/results.
-static bool
-scanframe(Stkframe *frame, void *wbufp)
-{
-	Func *f;
-	StackMap *stackmap;
-	BitVector bv;
-	uintptr size;
-	uintptr targetpc;
-	int32 pcdata;
-	bool afterprologue;
-	bool precise;
-
-	f = frame->fn;
-	targetpc = frame->continpc;
-	if(targetpc == 0) {
-		// Frame is dead.
-		return true;
-	}
-	if(targetpc != f->entry)
-		targetpc--;
-	pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc);
-	if(pcdata == -1) {
-		// We do not have a valid pcdata value but there might be a
-		// stackmap for this function.  It is likely that we are looking
-		// at the function prologue, assume so and hope for the best.
-		pcdata = 0;
-	}
-
-	// Scan local variables if stack frame has been allocated.
-	// Use pointer information if known.
-	afterprologue = (frame->varp > (byte*)frame->sp);
-	precise = false;
-	if(afterprologue) {
-		stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
-		if(stackmap == nil) {
-			// No locals information, scan everything.
-			size = frame->varp - (byte*)frame->sp;
-			if(Debug > 2)
-				runtime·printf("frame %s unsized locals %p+%p\n", runtime·funcname(f), frame->varp-size, size);
-			enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
-		} else if(stackmap->n < 0) {
-			// Locals size information, scan just the locals.
-			size = -stackmap->n;
-			if(Debug > 2)
-				runtime·printf("frame %s conservative locals %p+%p\n", runtime·funcname(f), frame->varp-size, size);
-			enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
-		} else if(stackmap->n > 0) {
-			// Locals bitmap information, scan just the pointers in
-			// locals.
-			if(pcdata < 0 || pcdata >= stackmap->n) {
-				// don't know where we are
-				runtime·printf("pcdata is %d and %d stack map entries for %s (targetpc=%p)\n",
-					pcdata, stackmap->n, runtime·funcname(f), targetpc);
-				runtime·throw("scanframe: bad symbol table");
-			}
-			bv = runtime·stackmapdata(stackmap, pcdata);
-			size = (bv.n * PtrSize) / BitsPerPointer;
-			precise = true;
-			scanbitvector(f, true, frame->varp - size, &bv, afterprologue, wbufp);
-		}
-	}
-
-	// Scan arguments.
-	// Use pointer information if known.
-	stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
-	if(stackmap != nil) {
-		bv = runtime·stackmapdata(stackmap, pcdata);
-		scanbitvector(f, precise, frame->argp, &bv, true, wbufp);
-	} else {
-		if(Debug > 2)
-			runtime·printf("frame %s conservative args %p+%p\n", runtime·funcname(f), frame->argp, (uintptr)frame->arglen);
-		enqueue1(wbufp, (Obj){frame->argp, frame->arglen, 0});
-	}
-	return true;
-}
-
-static void
-addstackroots(G *gp, Workbuf **wbufp)
-{
-	M *mp;
-	int32 n;
-	Stktop *stk;
-	uintptr sp, guard;
-	void *base;
-	uintptr size;
-
-	switch(gp->status){
-	default:
-		runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp->status, gp, gp->goid);
-		runtime·throw("mark - bad status");
-	case Gdead:
-		return;
-	case Grunning:
-		runtime·throw("mark - world not stopped");
-	case Grunnable:
-	case Gsyscall:
-	case Gwaiting:
-		break;
-	}
-
-	if(gp == g)
-		runtime·throw("can't scan our own stack");
-	if((mp = gp->m) != nil && mp->helpgc)
-		runtime·throw("can't scan gchelper stack");
-
-	if(gp->syscallstack != (uintptr)nil) {
-		// Scanning another goroutine that is about to enter or might
-		// have just exited a system call. It may be executing code such
-		// as schedlock and may have needed to start a new stack segment.
-		// Use the stack segment and stack pointer at the time of
-		// the system call instead, since that won't change underfoot.
-		sp = gp->syscallsp;
-		stk = (Stktop*)gp->syscallstack;
-		guard = gp->syscallguard;
-	} else {
-		// Scanning another goroutine's stack.
-		// The goroutine is usually asleep (the world is stopped).
-		sp = gp->sched.sp;
-		stk = (Stktop*)gp->stackbase;
-		guard = gp->stackguard;
-		// For function about to start, context argument is a root too.
-		if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
-			enqueue1(wbufp, (Obj){base, size, 0});
-	}
-	if(ScanStackByFrames) {
-		USED(sp);
-		USED(stk);
-		USED(guard);
-		runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, scanframe, wbufp, false);
-	} else {
-		n = 0;
-		while(stk) {
-			if(sp < guard-StackGuard || (uintptr)stk < sp) {
-				runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
-				runtime·throw("scanstack");
-			}
-			if(Debug > 2)
-				runtime·printf("conservative stack %p+%p\n", (byte*)sp, (uintptr)stk-sp);
-			enqueue1(wbufp, (Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
-			sp = stk->gobuf.sp;
-			guard = stk->stackguard;
-			stk = (Stktop*)stk->stackbase;
-			n++;
-		}
-	}
-}
-
-void
-runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot)
-{
-	FinBlock *block;
-	Finalizer *f;
-
-	runtime·lock(&finlock);
-	if(finq == nil || finq->cnt == finq->cap) {
-		if(finc == nil) {
-			finc = runtime·persistentalloc(FinBlockSize, 0, &mstats.gc_sys);
-			finc->cap = (FinBlockSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1;
-			finc->alllink = allfin;
-			allfin = finc;
-		}
-		block = finc;
-		finc = block->next;
-		block->next = finq;
-		finq = block;
-	}
-	f = &finq->fin[finq->cnt];
-	finq->cnt++;
-	f->fn = fn;
-	f->nret = nret;
-	f->fint = fint;
-	f->ot = ot;
-	f->arg = p;
-	runtime·fingwake = true;
-	runtime·unlock(&finlock);
-}
-
-void
-runtime·iterate_finq(void (*callback)(FuncVal*, byte*, uintptr, Type*, PtrType*))
-{
-	FinBlock *fb;
-	Finalizer *f;
-	uintptr i;
-
-	for(fb = allfin; fb; fb = fb->alllink) {
-		for(i = 0; i < fb->cnt; i++) {
-			f = &fb->fin[i];
-			callback(f->fn, f->arg, f->nret, f->fint, f->ot);
-		}
-	}
-}
-
-void
-runtime·MSpan_EnsureSwept(MSpan *s)
-{
-	uint32 sg;
-
-	// Caller must disable preemption.
-	// Otherwise when this function returns the span can become unswept again
-	// (if GC is triggered on another goroutine).
-	if(m->locks == 0 && m->mallocing == 0 && g != m->g0)
-		runtime·throw("MSpan_EnsureSwept: m is not locked");
-
-	sg = runtime·mheap.sweepgen;
-	if(runtime·atomicload(&s->sweepgen) == sg)
-		return;
-	if(runtime·cas(&s->sweepgen, sg-2, sg-1)) {
-		runtime·MSpan_Sweep(s);
-		return;
-	}
-	// unfortunate condition, and we don't have efficient means to wait
-	while(runtime·atomicload(&s->sweepgen) != sg)
-		runtime·osyield();  
-}
-
-// Sweep frees or collects finalizers for blocks not marked in the mark phase.
-// It clears the mark bits in preparation for the next GC round.
-// Returns true if the span was returned to heap.
-bool
-runtime·MSpan_Sweep(MSpan *s)
-{
-	int32 cl, n, npages, nfree;
-	uintptr size, off, *bitp, shift, bits;
-	uint32 sweepgen;
-	byte *p;
-	MCache *c;
-	byte *arena_start;
-	MLink head, *end;
-	byte *type_data;
-	byte compression;
-	uintptr type_data_inc;
-	MLink *x;
-	Special *special, **specialp, *y;
-	bool res, sweepgenset;
-
-	// It's critical that we enter this function with preemption disabled,
-	// GC must not start while we are in the middle of this function.
-	if(m->locks == 0 && m->mallocing == 0 && g != m->g0)
-		runtime·throw("MSpan_Sweep: m is not locked");
-	sweepgen = runtime·mheap.sweepgen;
-	if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) {
-		runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n",
-			s->state, s->sweepgen, sweepgen);
-		runtime·throw("MSpan_Sweep: bad span state");
-	}
-	arena_start = runtime·mheap.arena_start;
-	cl = s->sizeclass;
-	size = s->elemsize;
-	if(cl == 0) {
-		n = 1;
-	} else {
-		// Chunk full of small blocks.
-		npages = runtime·class_to_allocnpages[cl];
-		n = (npages << PageShift) / size;
-	}
-	res = false;
-	nfree = 0;
-	end = &head;
-	c = m->mcache;
-	sweepgenset = false;
-
-	// mark any free objects in this span so we don't collect them
-	for(x = s->freelist; x != nil; x = x->next) {
-		// This is markonly(x) but faster because we don't need
-		// atomic access and we're guaranteed to be pointing at
-		// the head of a valid object.
-		off = (uintptr*)x - (uintptr*)runtime·mheap.arena_start;
-		bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		*bitp |= bitMarked<<shift;
-	}
-
-	// Unlink & free special records for any objects we're about to free.
-	specialp = &s->specials;
-	special = *specialp;
-	while(special != nil) {
-		// A finalizer can be set for an inner byte of an object, find object beginning.
-		p = (byte*)(s->start << PageShift) + special->offset/size*size;
-		off = (uintptr*)p - (uintptr*)arena_start;
-		bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		bits = *bitp>>shift;
-		if((bits & (bitAllocated|bitMarked)) == bitAllocated) {
-			// Find the exact byte for which the special was setup
-			// (as opposed to object beginning).
-			p = (byte*)(s->start << PageShift) + special->offset;
-			// about to free object: splice out special record
-			y = special;
-			special = special->next;
-			*specialp = special;
-			if(!runtime·freespecial(y, p, size, false)) {
-				// stop freeing of object if it has a finalizer
-				*bitp |= bitMarked << shift;
-			}
-		} else {
-			// object is still live: keep special record
-			specialp = &special->next;
-			special = *specialp;
-		}
-	}
-
-	type_data = (byte*)s->types.data;
-	type_data_inc = sizeof(uintptr);
-	compression = s->types.compression;
-	switch(compression) {
-	case MTypes_Bytes:
-		type_data += 8*sizeof(uintptr);
-		type_data_inc = 1;
-		break;
-	}
-
-	// Sweep through n objects of given size starting at p.
-	// This thread owns the span now, so it can manipulate
-	// the block bitmap without atomic operations.
-	p = (byte*)(s->start << PageShift);
-	for(; n > 0; n--, p += size, type_data+=type_data_inc) {
-		off = (uintptr*)p - (uintptr*)arena_start;
-		bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		bits = *bitp>>shift;
-
-		if((bits & bitAllocated) == 0)
-			continue;
-
-		if((bits & bitMarked) != 0) {
-			*bitp &= ~(bitMarked<<shift);
-			continue;
-		}
-
-		if(runtime·debug.allocfreetrace)
-			runtime·tracefree(p, size);
-
-		// Clear mark and scan bits.
-		*bitp &= ~((bitScan|bitMarked)<<shift);
-
-		if(cl == 0) {
-			// Free large span.
-			runtime·unmarkspan(p, 1<<PageShift);
-			s->needzero = 1;
-			// important to set sweepgen before returning it to heap
-			runtime·atomicstore(&s->sweepgen, sweepgen);
-			sweepgenset = true;
-			// See note about SysFault vs SysFree in malloc.goc.
-			if(runtime·debug.efence)
-				runtime·SysFault(p, size);
-			else
-				runtime·MHeap_Free(&runtime·mheap, s, 1);
-			c->local_nlargefree++;
-			c->local_largefree += size;
-			runtime·xadd64(&mstats.next_gc, -(uint64)(size * (gcpercent + 100)/100));
-			res = true;
-		} else {
-			// Free small object.
-			switch(compression) {
-			case MTypes_Words:
-				*(uintptr*)type_data = 0;
-				break;
-			case MTypes_Bytes:
-				*(byte*)type_data = 0;
-				break;
-			}
-			if(size > 2*sizeof(uintptr))
-				((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll;	// mark as "needs to be zeroed"
-			else if(size > sizeof(uintptr))
-				((uintptr*)p)[1] = 0;
-
-			end->next = (MLink*)p;
-			end = (MLink*)p;
-			nfree++;
-		}
-	}
-
-	// We need to set s->sweepgen = h->sweepgen only when all blocks are swept,
-	// because of the potential for a concurrent free/SetFinalizer.
-	// But we need to set it before we make the span available for allocation
-	// (return it to heap or mcentral), because allocation code assumes that a
-	// span is already swept if available for allocation.
-
-	if(!sweepgenset && nfree == 0) {
-		// The span must be in our exclusive ownership until we update sweepgen,
-		// check for potential races.
-		if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) {
-			runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n",
-				s->state, s->sweepgen, sweepgen);
-			runtime·throw("MSpan_Sweep: bad span state after sweep");
-		}
-		runtime·atomicstore(&s->sweepgen, sweepgen);
-	}
-	if(nfree > 0) {
-		c->local_nsmallfree[cl] += nfree;
-		c->local_cachealloc -= nfree * size;
-		runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (gcpercent + 100)/100));
-		res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end);
-		//MCentral_FreeSpan updates sweepgen
-	}
-	return res;
-}
-
-// State of background sweep.
-// Pretected by gclock.
-static struct
-{
-	G*	g;
-	bool	parked;
-
-	MSpan**	spans;
-	uint32	nspan;
-	uint32	spanidx;
-} sweep;
-
-// background sweeping goroutine
-static void
-bgsweep(void)
-{
-	g->issystem = 1;
-	for(;;) {
-		while(runtime·sweepone() != -1) {
-			gcstats.nbgsweep++;
-			runtime·gosched();
-		}
-		runtime·lock(&gclock);
-		if(!runtime·mheap.sweepdone) {
-			// It's possible if GC has happened between sweepone has
-			// returned -1 and gclock lock.
-			runtime·unlock(&gclock);
-			continue;
-		}
-		sweep.parked = true;
-		g->isbackground = true;
-		runtime·parkunlock(&gclock, "GC sweep wait");
-		g->isbackground = false;
-	}
-}
-
-// sweeps one span
-// returns number of pages returned to heap, or -1 if there is nothing to sweep
-uintptr
-runtime·sweepone(void)
-{
-	MSpan *s;
-	uint32 idx, sg;
-	uintptr npages;
-
-	// increment locks to ensure that the goroutine is not preempted
-	// in the middle of sweep thus leaving the span in an inconsistent state for next GC
-	m->locks++;
-	sg = runtime·mheap.sweepgen;
-	for(;;) {
-		idx = runtime·xadd(&sweep.spanidx, 1) - 1;
-		if(idx >= sweep.nspan) {
-			runtime·mheap.sweepdone = true;
-			m->locks--;
-			return -1;
-		}
-		s = sweep.spans[idx];
-		if(s->state != MSpanInUse) {
-			s->sweepgen = sg;
-			continue;
-		}
-		if(s->sweepgen != sg-2 || !runtime·cas(&s->sweepgen, sg-2, sg-1))
-			continue;
-		if(s->incache)
-			runtime·throw("sweep of incache span");
-		npages = s->npages;
-		if(!runtime·MSpan_Sweep(s))
-			npages = 0;
-		m->locks--;
-		return npages;
-	}
-}
-
-static void
-dumpspan(uint32 idx)
-{
-	int32 sizeclass, n, npages, i, column;
-	uintptr size;
-	byte *p;
-	byte *arena_start;
-	MSpan *s;
-	bool allocated;
-
-	s = runtime·mheap.allspans[idx];
-	if(s->state != MSpanInUse)
-		return;
-	arena_start = runtime·mheap.arena_start;
-	p = (byte*)(s->start << PageShift);
-	sizeclass = s->sizeclass;
-	size = s->elemsize;
-	if(sizeclass == 0) {
-		n = 1;
-	} else {
-		npages = runtime·class_to_allocnpages[sizeclass];
-		n = (npages << PageShift) / size;
-	}
-	
-	runtime·printf("%p .. %p:\n", p, p+n*size);
-	column = 0;
-	for(; n>0; n--, p+=size) {
-		uintptr off, *bitp, shift, bits;
-
-		off = (uintptr*)p - (uintptr*)arena_start;
-		bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		bits = *bitp>>shift;
-
-		allocated = ((bits & bitAllocated) != 0);
-
-		for(i=0; i<size; i+=sizeof(void*)) {
-			if(column == 0) {
-				runtime·printf("\t");
-			}
-			if(i == 0) {
-				runtime·printf(allocated ? "(" : "[");
-				runtime·printf("%p: ", p+i);
-			} else {
-				runtime·printf(" ");
-			}
-
-			runtime·printf("%p", *(void**)(p+i));
-
-			if(i+sizeof(void*) >= size) {
-				runtime·printf(allocated ? ") " : "] ");
-			}
-
-			column++;
-			if(column == 8) {
-				runtime·printf("\n");
-				column = 0;
-			}
-		}
-	}
-	runtime·printf("\n");
-}
-
-// A debugging function to dump the contents of memory
-void
-runtime·memorydump(void)
-{
-	uint32 spanidx;
-
-	for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
-		dumpspan(spanidx);
-	}
-}
-
-void
-runtime·gchelper(void)
-{
-	uint32 nproc;
-
-	m->traceback = 2;
-	gchelperstart();
-
-	// parallel mark for over gc roots
-	runtime·parfordo(work.markfor);
-
-	// help other threads scan secondary blocks
-	scanblock(nil, true);
-
-	bufferList[m->helpgc].busy = 0;
-	nproc = work.nproc;  // work.nproc can change right after we increment work.ndone
-	if(runtime·xadd(&work.ndone, +1) == nproc-1)
-		runtime·notewakeup(&work.alldone);
-	m->traceback = 0;
-}
-
-static void
-cachestats(void)
-{
-	MCache *c;
-	P *p, **pp;
-
-	for(pp=runtime·allp; p=*pp; pp++) {
-		c = p->mcache;
-		if(c==nil)
-			continue;
-		runtime·purgecachedstats(c);
-	}
-}
-
-static void
-flushallmcaches(void)
-{
-	P *p, **pp;
-	MCache *c;
-
-	// Flush MCache's to MCentral.
-	for(pp=runtime·allp; p=*pp; pp++) {
-		c = p->mcache;
-		if(c==nil)
-			continue;
-		runtime·MCache_ReleaseAll(c);
-	}
-}
-
-void
-runtime·updatememstats(GCStats *stats)
-{
-	M *mp;
-	MSpan *s;
-	int32 i;
-	uint64 stacks_inuse, smallfree;
-	uint64 *src, *dst;
-
-	if(stats)
-		runtime·memclr((byte*)stats, sizeof(*stats));
-	stacks_inuse = 0;
-	for(mp=runtime·allm; mp; mp=mp->alllink) {
-		stacks_inuse += mp->stackinuse*FixedStack;
-		if(stats) {
-			src = (uint64*)&mp->gcstats;
-			dst = (uint64*)stats;
-			for(i=0; i<sizeof(*stats)/sizeof(uint64); i++)
-				dst[i] += src[i];
-			runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
-		}
-	}
-	mstats.stacks_inuse = stacks_inuse;
-	mstats.mcache_inuse = runtime·mheap.cachealloc.inuse;
-	mstats.mspan_inuse = runtime·mheap.spanalloc.inuse;
-	mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys +
-		mstats.mcache_sys + mstats.buckhash_sys + mstats.gc_sys + mstats.other_sys;
-	
-	// Calculate memory allocator stats.
-	// During program execution we only count number of frees and amount of freed memory.
-	// Current number of alive object in the heap and amount of alive heap memory
-	// are calculated by scanning all spans.
-	// Total number of mallocs is calculated as number of frees plus number of alive objects.
-	// Similarly, total amount of allocated memory is calculated as amount of freed memory
-	// plus amount of alive heap memory.
-	mstats.alloc = 0;
-	mstats.total_alloc = 0;
-	mstats.nmalloc = 0;
-	mstats.nfree = 0;
-	for(i = 0; i < nelem(mstats.by_size); i++) {
-		mstats.by_size[i].nmalloc = 0;
-		mstats.by_size[i].nfree = 0;
-	}
-
-	// Flush MCache's to MCentral.
-	flushallmcaches();
-
-	// Aggregate local stats.
-	cachestats();
-
-	// Scan all spans and count number of alive objects.
-	for(i = 0; i < runtime·mheap.nspan; i++) {
-		s = runtime·mheap.allspans[i];
-		if(s->state != MSpanInUse)
-			continue;
-		if(s->sizeclass == 0) {
-			mstats.nmalloc++;
-			mstats.alloc += s->elemsize;
-		} else {
-			mstats.nmalloc += s->ref;
-			mstats.by_size[s->sizeclass].nmalloc += s->ref;
-			mstats.alloc += s->ref*s->elemsize;
-		}
-	}
-
-	// Aggregate by size class.
-	smallfree = 0;
-	mstats.nfree = runtime·mheap.nlargefree;
-	for(i = 0; i < nelem(mstats.by_size); i++) {
-		mstats.nfree += runtime·mheap.nsmallfree[i];
-		mstats.by_size[i].nfree = runtime·mheap.nsmallfree[i];
-		mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i];
-		smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i];
-	}
-	mstats.nmalloc += mstats.nfree;
-
-	// Calculate derived stats.
-	mstats.total_alloc = mstats.alloc + runtime·mheap.largefree + smallfree;
-	mstats.heap_alloc = mstats.alloc;
-	mstats.heap_objects = mstats.nmalloc - mstats.nfree;
-}
-
-// Structure of arguments passed to function gc().
-// This allows the arguments to be passed via runtime·mcall.
-struct gc_args
-{
-	int64 start_time; // start time of GC in ns (just before stoptheworld)
-	bool  eagersweep;
-};
-
-static void gc(struct gc_args *args);
-static void mgc(G *gp);
-
-static int32
-readgogc(void)
-{
-	byte *p;
-
-	p = runtime·getenv("GOGC");
-	if(p == nil || p[0] == '\0')
-		return 100;
-	if(runtime·strcmp(p, (byte*)"off") == 0)
-		return -1;
-	return runtime·atoi(p);
-}
-
-// force = 1 - do GC regardless of current heap usage
-// force = 2 - go GC and eager sweep
-void
-runtime·gc(int32 force)
-{
-	struct gc_args a;
-	int32 i;
-
-	// The atomic operations are not atomic if the uint64s
-	// are not aligned on uint64 boundaries. This has been
-	// a problem in the past.
-	if((((uintptr)&work.empty) & 7) != 0)
-		runtime·throw("runtime: gc work buffer is misaligned");
-	if((((uintptr)&work.full) & 7) != 0)
-		runtime·throw("runtime: gc work buffer is misaligned");
-
-	// The gc is turned off (via enablegc) until
-	// the bootstrap has completed.
-	// Also, malloc gets called in the guts
-	// of a number of libraries that might be
-	// holding locks.  To avoid priority inversion
-	// problems, don't bother trying to run gc
-	// while holding a lock.  The next mallocgc
-	// without a lock will do the gc instead.
-	if(!mstats.enablegc || g == m->g0 || m->locks > 0 || runtime·panicking)
-		return;
-
-	if(gcpercent == GcpercentUnknown) {	// first time through
-		runtime·lock(&runtime·mheap);
-		if(gcpercent == GcpercentUnknown)
-			gcpercent = readgogc();
-		runtime·unlock(&runtime·mheap);
-	}
-	if(gcpercent < 0)
-		return;
-
-	runtime·semacquire(&runtime·worldsema, false);
-	if(force==0 && mstats.heap_alloc < mstats.next_gc) {
-		// typically threads which lost the race to grab
-		// worldsema exit here when gc is done.
-		runtime·semrelease(&runtime·worldsema);
-		return;
-	}
-
-	// Ok, we're doing it!  Stop everybody else
-	a.start_time = runtime·nanotime();
-	a.eagersweep = force >= 2;
-	m->gcing = 1;
-	runtime·stoptheworld();
-	
-	clearpools();
-
-	// Run gc on the g0 stack.  We do this so that the g stack
-	// we're currently running on will no longer change.  Cuts
-	// the root set down a bit (g0 stacks are not scanned, and
-	// we don't need to scan gc's internal state).  Also an
-	// enabler for copyable stacks.
-	for(i = 0; i < (runtime·debug.gctrace > 1 ? 2 : 1); i++) {
-		if(i > 0)
-			a.start_time = runtime·nanotime();
-		// switch to g0, call gc(&a), then switch back
-		g->param = &a;
-		g->status = Gwaiting;
-		g->waitreason = "garbage collection";
-		runtime·mcall(mgc);
-	}
-
-	// all done
-	m->gcing = 0;
-	m->locks++;
-	runtime·semrelease(&runtime·worldsema);
-	runtime·starttheworld();
-	m->locks--;
-
-	// now that gc is done, kick off finalizer thread if needed
-	if(!ConcurrentSweep) {
-		// give the queued finalizers, if any, a chance to run
-		runtime·gosched();
-	}
-}
-
-static void
-mgc(G *gp)
-{
-	gc(gp->param);
-	gp->param = nil;
-	gp->status = Grunning;
-	runtime·gogo(&gp->sched);
-}
-
-static void
-gc(struct gc_args *args)
-{
-	int64 t0, t1, t2, t3, t4;
-	uint64 heap0, heap1, obj, ninstr;
-	GCStats stats;
-	uint32 i;
-	Eface eface;
-
-	if(runtime·debug.allocfreetrace)
-		runtime·tracegc();
-
-	m->traceback = 2;
-	t0 = args->start_time;
-	work.tstart = args->start_time; 
-
-	if(CollectStats)
-		runtime·memclr((byte*)&gcstats, sizeof(gcstats));
-
-	m->locks++;	// disable gc during mallocs in parforalloc
-	if(work.markfor == nil)
-		work.markfor = runtime·parforalloc(MaxGcproc);
-	m->locks--;
-
-	if(itabtype == nil) {
-		// get C pointer to the Go type "itab"
-		runtime·gc_itab_ptr(&eface);
-		itabtype = ((PtrType*)eface.type)->elem;
-	}
-
-	t1 = 0;
-	if(runtime·debug.gctrace)
-		t1 = runtime·nanotime();
-
-	// Sweep what is not sweeped by bgsweep.
-	while(runtime·sweepone() != -1)
-		gcstats.npausesweep++;
-
-	work.nwait = 0;
-	work.ndone = 0;
-	work.nproc = runtime·gcprocs();
-	runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allglen, nil, false, markroot);
-	if(work.nproc > 1) {
-		runtime·noteclear(&work.alldone);
-		runtime·helpgc(work.nproc);
-	}
-
-	t2 = 0;
-	if(runtime·debug.gctrace)
-		t2 = runtime·nanotime();
-
-	gchelperstart();
-	runtime·parfordo(work.markfor);
-	scanblock(nil, true);
-
-	t3 = 0;
-	if(runtime·debug.gctrace)
-		t3 = runtime·nanotime();
-
-	bufferList[m->helpgc].busy = 0;
-	if(work.nproc > 1)
-		runtime·notesleep(&work.alldone);
-
-	cachestats();
-	// next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
-	// estimate what was live heap size after previous GC (for tracing only)
-	heap0 = mstats.next_gc*100/(gcpercent+100);
-	// conservatively set next_gc to high value assuming that everything is live
-	// concurrent/lazy sweep will reduce this number while discovering new garbage
-	mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
-
-	t4 = runtime·nanotime();
-	mstats.last_gc = runtime·unixnanotime();  // must be Unix time to make sense to user
-	mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0;
-	mstats.pause_total_ns += t4 - t0;
-	mstats.numgc++;
-	if(mstats.debuggc)
-		runtime·printf("pause %D\n", t4-t0);
-
-	if(runtime·debug.gctrace) {
-		heap1 = mstats.heap_alloc;
-		runtime·updatememstats(&stats);
-		if(heap1 != mstats.heap_alloc) {
-			runtime·printf("runtime: mstats skew: heap=%D/%D\n", heap1, mstats.heap_alloc);
-			runtime·throw("mstats skew");
-		}
-		obj = mstats.nmalloc - mstats.nfree;
-
-		stats.nprocyield += work.markfor->nprocyield;
-		stats.nosyield += work.markfor->nosyield;
-		stats.nsleep += work.markfor->nsleep;
-
-		runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects,"
-				" %d/%d/%d sweeps,"
-				" %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
-			mstats.numgc, work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t3-t2)/1000, (t4-t3)/1000,
-			heap0>>20, heap1>>20, obj,
-			mstats.nmalloc, mstats.nfree,
-			sweep.nspan, gcstats.nbgsweep, gcstats.npausesweep,
-			stats.nhandoff, stats.nhandoffcnt,
-			work.markfor->nsteal, work.markfor->nstealcnt,
-			stats.nprocyield, stats.nosyield, stats.nsleep);
-		gcstats.nbgsweep = gcstats.npausesweep = 0;
-		if(CollectStats) {
-			runtime·printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n",
-				gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.notype, gcstats.obj.typelookup);
-			if(gcstats.ptr.cnt != 0)
-				runtime·printf("avg ptrbufsize: %D (%D/%D)\n",
-					gcstats.ptr.sum/gcstats.ptr.cnt, gcstats.ptr.sum, gcstats.ptr.cnt);
-			if(gcstats.obj.cnt != 0)
-				runtime·printf("avg nobj: %D (%D/%D)\n",
-					gcstats.obj.sum/gcstats.obj.cnt, gcstats.obj.sum, gcstats.obj.cnt);
-			runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan, gcstats.rescanbytes);
-
-			runtime·printf("instruction counts:\n");
-			ninstr = 0;
-			for(i=0; i<nelem(gcstats.instr); i++) {
-				runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i]);
-				ninstr += gcstats.instr[i];
-			}
-			runtime·printf("\ttotal:\t%D\n", ninstr);
-
-			runtime·printf("putempty: %D, getfull: %D\n", gcstats.putempty, gcstats.getfull);
-
-			runtime·printf("markonly base lookup: bit %D word %D span %D\n", gcstats.markonly.foundbit, gcstats.markonly.foundword, gcstats.markonly.foundspan);
-			runtime·printf("flushptrbuf base lookup: bit %D word %D span %D\n", gcstats.flushptrbuf.foundbit, gcstats.flushptrbuf.foundword, gcstats.flushptrbuf.foundspan);
-		}
-	}
-
-	// We cache current runtime·mheap.allspans array in sweep.spans,
-	// because the former can be resized and freed.
-	// Otherwise we would need to take heap lock every time
-	// we want to convert span index to span pointer.
-
-	// Free the old cached array if necessary.
-	if(sweep.spans && sweep.spans != runtime·mheap.allspans)
-		runtime·SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &mstats.other_sys);
-	// Cache the current array.
-	runtime·mheap.sweepspans = runtime·mheap.allspans;
-	runtime·mheap.sweepgen += 2;
-	runtime·mheap.sweepdone = false;
-	sweep.spans = runtime·mheap.allspans;
-	sweep.nspan = runtime·mheap.nspan;
-	sweep.spanidx = 0;
-
-	// Temporary disable concurrent sweep, because we see failures on builders.
-	if(ConcurrentSweep && !args->eagersweep) {
-		runtime·lock(&gclock);
-		if(sweep.g == nil)
-			sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, runtime·gc);
-		else if(sweep.parked) {
-			sweep.parked = false;
-			runtime·ready(sweep.g);
-		}
-		runtime·unlock(&gclock);
-	} else {
-		// Sweep all spans eagerly.
-		while(runtime·sweepone() != -1)
-			gcstats.npausesweep++;
-	}
-
-	// Shrink a stack if not much of it is being used.
-	// TODO: do in a parfor
-	for(i = 0; i < runtime·allglen; i++)
-		runtime·shrinkstack(runtime·allg[i]);
-
-	runtime·MProf_GC();
-	m->traceback = 0;
-}
-
-extern uintptr runtime·sizeof_C_MStats;
-
-void
-runtime·ReadMemStats(MStats *stats)
-{
-	// Have to acquire worldsema to stop the world,
-	// because stoptheworld can only be used by
-	// one goroutine at a time, and there might be
-	// a pending garbage collection already calling it.
-	runtime·semacquire(&runtime·worldsema, false);
-	m->gcing = 1;
-	runtime·stoptheworld();
-	runtime·updatememstats(nil);
-	// Size of the trailing by_size array differs between Go and C,
-	// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
-	runtime·memcopy(runtime·sizeof_C_MStats, stats, &mstats);
-	m->gcing = 0;
-	m->locks++;
-	runtime·semrelease(&runtime·worldsema);
-	runtime·starttheworld();
-	m->locks--;
-}
-
-void
-runtime∕debug·readGCStats(Slice *pauses)
-{
-	uint64 *p;
-	uint32 i, n;
-
-	// Calling code in runtime/debug should make the slice large enough.
-	if(pauses->cap < nelem(mstats.pause_ns)+3)
-		runtime·throw("runtime: short slice passed to readGCStats");
-
-	// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
-	p = (uint64*)pauses->array;
-	runtime·lock(&runtime·mheap);
-	n = mstats.numgc;
-	if(n > nelem(mstats.pause_ns))
-		n = nelem(mstats.pause_ns);
-	
-	// The pause buffer is circular. The most recent pause is at
-	// pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
-	// from there to go back farther in time. We deliver the times
-	// most recent first (in p[0]).
-	for(i=0; i<n; i++)
-		p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)];
-
-	p[n] = mstats.last_gc;
-	p[n+1] = mstats.numgc;
-	p[n+2] = mstats.pause_total_ns;	
-	runtime·unlock(&runtime·mheap);
-	pauses->len = n+3;
-}
-
-int32
-runtime·setgcpercent(int32 in) {
-	int32 out;
-
-	runtime·lock(&runtime·mheap);
-	if(gcpercent == GcpercentUnknown)
-		gcpercent = readgogc();
-	out = gcpercent;
-	if(in < 0)
-		in = -1;
-	gcpercent = in;
-	runtime·unlock(&runtime·mheap);
-	return out;
-}
-
-static void
-gchelperstart(void)
-{
-	if(m->helpgc < 0 || m->helpgc >= MaxGcproc)
-		runtime·throw("gchelperstart: bad m->helpgc");
-	if(runtime·xchg(&bufferList[m->helpgc].busy, 1))
-		runtime·throw("gchelperstart: already busy");
-	if(g != m->g0)
-		runtime·throw("gchelper not running on g0 stack");
-}
-
-static void
-runfinq(void)
-{
-	Finalizer *f;
-	FinBlock *fb, *next;
-	byte *frame;
-	uint32 framesz, framecap, i;
-	Eface *ef, ef1;
-
-	// This function blocks for long periods of time, and because it is written in C
-	// we have no liveness information. Zero everything so that uninitialized pointers
-	// do not cause memory leaks.
-	f = nil;
-	fb = nil;
-	next = nil;
-	frame = nil;
-	framecap = 0;
-	framesz = 0;
-	i = 0;
-	ef = nil;
-	ef1.type = nil;
-	ef1.data = nil;
-	
-	// force flush to memory
-	USED(&f);
-	USED(&fb);
-	USED(&next);
-	USED(&framesz);
-	USED(&i);
-	USED(&ef);
-	USED(&ef1);
-
-	for(;;) {
-		runtime·lock(&finlock);
-		fb = finq;
-		finq = nil;
-		if(fb == nil) {
-			runtime·fingwait = true;
-			g->isbackground = true;
-			runtime·parkunlock(&finlock, "finalizer wait");
-			g->isbackground = false;
-			continue;
-		}
-		runtime·unlock(&finlock);
-		if(raceenabled)
-			runtime·racefingo();
-		for(; fb; fb=next) {
-			next = fb->next;
-			for(i=0; i<fb->cnt; i++) {
-				f = &fb->fin[i];
-				framesz = sizeof(Eface) + f->nret;
-				if(framecap < framesz) {
-					runtime·free(frame);
-					// The frame does not contain pointers interesting for GC,
-					// all not yet finalized objects are stored in finq.
-					// If we do not mark it as FlagNoScan,
-					// the last finalized object is not collected.
-					frame = runtime·mallocgc(framesz, 0, FlagNoScan|FlagNoInvokeGC);
-					framecap = framesz;
-				}
-				if(f->fint == nil)
-					runtime·throw("missing type in runfinq");
-				if(f->fint->kind == KindPtr) {
-					// direct use of pointer
-					*(void**)frame = f->arg;
-				} else if(((InterfaceType*)f->fint)->mhdr.len == 0) {
-					// convert to empty interface
-					ef = (Eface*)frame;
-					ef->type = f->ot;
-					ef->data = f->arg;
-				} else {
-					// convert to interface with methods, via empty interface.
-					ef1.type = f->ot;
-					ef1.data = f->arg;
-					if(!runtime·ifaceE2I2((InterfaceType*)f->fint, ef1, (Iface*)frame))
-						runtime·throw("invalid type conversion in runfinq");
-				}
-				reflect·call(f->fn, frame, framesz, framesz);
-				f->fn = nil;
-				f->arg = nil;
-				f->ot = nil;
-			}
-			fb->cnt = 0;
-			runtime·lock(&finlock);
-			fb->next = finc;
-			finc = fb;
-			runtime·unlock(&finlock);
-		}
-
-		// Zero everything that's dead, to avoid memory leaks.
-		// See comment at top of function.
-		f = nil;
-		fb = nil;
-		next = nil;
-		i = 0;
-		ef = nil;
-		ef1.type = nil;
-		ef1.data = nil;
-		runtime·gc(1);	// trigger another gc to clean up the finalized objects, if possible
-	}
-}
-
-void
-runtime·createfing(void)
-{
-	if(fing != nil)
-		return;
-	// Here we use gclock instead of finlock,
-	// because newproc1 can allocate, which can cause on-demand span sweep,
-	// which can queue finalizers, which would deadlock.
-	runtime·lock(&gclock);
-	if(fing == nil)
-		fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
-	runtime·unlock(&gclock);
-}
-
-G*
-runtime·wakefing(void)
-{
-	G *res;
-
-	res = nil;
-	runtime·lock(&finlock);
-	if(runtime·fingwait && runtime·fingwake) {
-		runtime·fingwait = false;
-		runtime·fingwake = false;
-		res = fing;
-	}
-	runtime·unlock(&finlock);
-	return res;
-}
-
-void
-runtime·marknogc(void *v)
-{
-	uintptr *b, off, shift;
-
-	off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
-	b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	shift = off % wordsPerBitmapWord;
-	*b = (*b & ~(bitAllocated<<shift)) | bitBlockBoundary<<shift;
-}
-
-void
-runtime·markscan(void *v)
-{
-	uintptr *b, off, shift;
-
-	off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
-	b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	shift = off % wordsPerBitmapWord;
-	*b |= bitScan<<shift;
-}
-
-// mark the block at v as freed.
-void
-runtime·markfreed(void *v)
-{
-	uintptr *b, off, shift;
-
-	if(0)
-		runtime·printf("markfreed %p\n", v);
-
-	if((byte*)v > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
-		runtime·throw("markfreed: bad pointer");
-
-	off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
-	b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	shift = off % wordsPerBitmapWord;
-	*b = (*b & ~(bitMask<<shift)) | (bitAllocated<<shift);
-}
-
-// check that the block at v of size n is marked freed.
-void
-runtime·checkfreed(void *v, uintptr n)
-{
-	uintptr *b, bits, off, shift;
-
-	if(!runtime·checking)
-		return;
-
-	if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
-		return;	// not allocated, so okay
-
-	off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;  // word offset
-	b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	shift = off % wordsPerBitmapWord;
-
-	bits = *b>>shift;
-	if((bits & bitAllocated) != 0) {
-		runtime·printf("checkfreed %p+%p: off=%p have=%p\n",
-			v, n, off, bits & bitMask);
-		runtime·throw("checkfreed: not freed");
-	}
-}
-
-// mark the span of memory at v as having n blocks of the given size.
-// if leftover is true, there is left over space at the end of the span.
-void
-runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
-{
-	uintptr *b, *b0, off, shift, i, x;
-	byte *p;
-
-	if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
-		runtime·throw("markspan: bad pointer");
-
-	if(runtime·checking) {
-		// bits should be all zero at the start
-		off = (byte*)v + size - runtime·mheap.arena_start;
-		b = (uintptr*)(runtime·mheap.arena_start - off/wordsPerBitmapWord);
-		for(i = 0; i < size/PtrSize/wordsPerBitmapWord; i++) {
-			if(b[i] != 0)
-				runtime·throw("markspan: span bits not zero");
-		}
-	}
-
-	p = v;
-	if(leftover)	// mark a boundary just past end of last block too
-		n++;
-
-	b0 = nil;
-	x = 0;
-	for(; n-- > 0; p += size) {
-		// Okay to use non-atomic ops here, because we control
-		// the entire span, and each bitmap word has bits for only
-		// one span, so no other goroutines are changing these
-		// bitmap words.
-		off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start;  // word offset
-		b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-		shift = off % wordsPerBitmapWord;
-		if(b0 != b) {
-			if(b0 != nil)
-				*b0 = x;
-			b0 = b;
-			x = 0;
-		}
-		x |= bitAllocated<<shift;
-	}
-	*b0 = x;
-}
-
-// unmark the span of memory at v of length n bytes.
-void
-runtime·unmarkspan(void *v, uintptr n)
-{
-	uintptr *p, *b, off;
-
-	if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
-		runtime·throw("markspan: bad pointer");
-
-	p = v;
-	off = p - (uintptr*)runtime·mheap.arena_start;  // word offset
-	if(off % wordsPerBitmapWord != 0)
-		runtime·throw("markspan: unaligned pointer");
-	b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
-	n /= PtrSize;
-	if(n%wordsPerBitmapWord != 0)
-		runtime·throw("unmarkspan: unaligned length");
-	// Okay to use non-atomic ops here, because we control
-	// the entire span, and each bitmap word has bits for only
-	// one span, so no other goroutines are changing these
-	// bitmap words.
-	n /= wordsPerBitmapWord;
-	while(n-- > 0)
-		*b-- = 0;
-}
-
-void
-runtime·MHeap_MapBits(MHeap *h)
-{
-	// Caller has added extra mappings to the arena.
-	// Add extra mappings of bitmap words as needed.
-	// We allocate extra bitmap pieces in chunks of bitmapChunk.
-	enum {
-		bitmapChunk = 8192
-	};
-	uintptr n;
-
-	n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
-	n = ROUND(n, bitmapChunk);
-	n = ROUND(n, PhysPageSize);
-	if(h->bitmap_mapped >= n)
-		return;
-
-	runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserved, &mstats.gc_sys);
-	h->bitmap_mapped = n;
-}