summaryrefslogtreecommitdiff
path: root/src/pkg/runtime/mgc0.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/runtime/mgc0.c')
-rw-r--r--src/pkg/runtime/mgc0.c2892
1 files changed, 0 insertions, 2892 deletions
diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
deleted file mode 100644
index 392da535b..000000000
--- a/src/pkg/runtime/mgc0.c
+++ /dev/null
@@ -1,2892 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Garbage collector (GC).
-//
-// GC is:
-// - mark&sweep
-// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
-// - parallel (up to MaxGcproc threads)
-// - partially concurrent (mark is stop-the-world, while sweep is concurrent)
-// - non-moving/non-compacting
-// - full (non-partial)
-//
-// GC rate.
-// Next GC is after we've allocated an extra amount of memory proportional to
-// the amount already in use. The proportion is controlled by GOGC environment variable
-// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
-// (this mark is tracked in next_gc variable). This keeps the GC cost in linear
-// proportion to the allocation cost. Adjusting GOGC just changes the linear constant
-// (and also the amount of extra memory used).
-//
-// Concurrent sweep.
-// The sweep phase proceeds concurrently with normal program execution.
-// The heap is swept span-by-span both lazily (when a goroutine needs another span)
-// and concurrently in a background goroutine (this helps programs that are not CPU bound).
-// However, at the end of the stop-the-world GC phase we don't know the size of the live heap,
-// and so next_gc calculation is tricky and happens as follows.
-// At the end of the stop-the-world phase next_gc is conservatively set based on total
-// heap size; all spans are marked as "needs sweeping".
-// Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory.
-// The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc
-// closer to the target value. However, this is not enough to avoid over-allocating memory.
-// Consider that a goroutine wants to allocate a new span for a large object and
-// there are no free swept spans, but there are small-object unswept spans.
-// If the goroutine naively allocates a new span, it can surpass the yet-unknown
-// target next_gc value. In order to prevent such cases (1) when a goroutine needs
-// to allocate a new small-object span, it sweeps small-object spans for the same
-// object size until it frees at least one object; (2) when a goroutine needs to
-// allocate large-object span from heap, it sweeps spans until it frees at least
-// that many pages into heap. Together these two measures ensure that we don't surpass
-// target next_gc value by a large margin. There is an exception: if a goroutine sweeps
-// and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span,
-// but there can still be other one-page unswept spans which could be combined into a two-page span.
-// It's critical to ensure that no operations proceed on unswept spans (that would corrupt
-// mark bits in GC bitmap). During GC all mcaches are flushed into the central cache,
-// so they are empty. When a goroutine grabs a new span into mcache, it sweeps it.
-// When a goroutine explicitly frees an object or sets a finalizer, it ensures that
-// the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish).
-// The finalizer goroutine is kicked off only when all spans are swept.
-// When the next GC starts, it sweeps all not-yet-swept spans (if any).
-
-#include "runtime.h"
-#include "arch_GOARCH.h"
-#include "malloc.h"
-#include "stack.h"
-#include "mgc0.h"
-#include "chan.h"
-#include "race.h"
-#include "type.h"
-#include "typekind.h"
-#include "funcdata.h"
-#include "../../cmd/ld/textflag.h"
-
-enum {
- Debug = 0,
- CollectStats = 0,
- ConcurrentSweep = 1,
-
- WorkbufSize = 16*1024,
- FinBlockSize = 4*1024,
-
- handoffThreshold = 4,
- IntermediateBufferCapacity = 64,
-
- // Bits in type information
- PRECISE = 1,
- LOOP = 2,
- PC_BITS = PRECISE | LOOP,
-
- RootData = 0,
- RootBss = 1,
- RootFinalizers = 2,
- RootSpanTypes = 3,
- RootFlushCaches = 4,
- RootCount = 5,
-};
-
-#define GcpercentUnknown (-2)
-
-// Initialized from $GOGC. GOGC=off means no gc.
-static int32 gcpercent = GcpercentUnknown;
-
-static FuncVal* poolcleanup;
-
-void
-sync·runtime_registerPoolCleanup(FuncVal *f)
-{
- poolcleanup = f;
-}
-
-static void
-clearpools(void)
-{
- P *p, **pp;
- MCache *c;
- int32 i;
-
- // clear sync.Pool's
- if(poolcleanup != nil)
- reflect·call(poolcleanup, nil, 0, 0);
-
- for(pp=runtime·allp; p=*pp; pp++) {
- // clear tinyalloc pool
- c = p->mcache;
- if(c != nil) {
- c->tiny = nil;
- c->tinysize = 0;
- }
- // clear defer pools
- for(i=0; i<nelem(p->deferpool); i++)
- p->deferpool[i] = nil;
- }
-}
-
-// Holding worldsema grants an M the right to try to stop the world.
-// The procedure is:
-//
-// runtime·semacquire(&runtime·worldsema);
-// m->gcing = 1;
-// runtime·stoptheworld();
-//
-// ... do stuff ...
-//
-// m->gcing = 0;
-// runtime·semrelease(&runtime·worldsema);
-// runtime·starttheworld();
-//
-uint32 runtime·worldsema = 1;
-
-typedef struct Obj Obj;
-struct Obj
-{
- byte *p; // data pointer
- uintptr n; // size of data in bytes
- uintptr ti; // type info
-};
-
-typedef struct Workbuf Workbuf;
-struct Workbuf
-{
-#define SIZE (WorkbufSize-sizeof(LFNode)-sizeof(uintptr))
- LFNode node; // must be first
- uintptr nobj;
- Obj obj[SIZE/sizeof(Obj) - 1];
- uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)];
-#undef SIZE
-};
-
-typedef struct Finalizer Finalizer;
-struct Finalizer
-{
- FuncVal *fn;
- void *arg;
- uintptr nret;
- Type *fint;
- PtrType *ot;
-};
-
-typedef struct FinBlock FinBlock;
-struct FinBlock
-{
- FinBlock *alllink;
- FinBlock *next;
- int32 cnt;
- int32 cap;
- Finalizer fin[1];
-};
-
-extern byte data[];
-extern byte edata[];
-extern byte bss[];
-extern byte ebss[];
-
-extern byte gcdata[];
-extern byte gcbss[];
-
-static Lock finlock; // protects the following variables
-static FinBlock *finq; // list of finalizers that are to be executed
-static FinBlock *finc; // cache of free blocks
-static FinBlock *allfin; // list of all blocks
-bool runtime·fingwait;
-bool runtime·fingwake;
-
-static Lock gclock;
-static G* fing;
-
-static void runfinq(void);
-static void bgsweep(void);
-static Workbuf* getempty(Workbuf*);
-static Workbuf* getfull(Workbuf*);
-static void putempty(Workbuf*);
-static Workbuf* handoff(Workbuf*);
-static void gchelperstart(void);
-static void flushallmcaches(void);
-static bool scanframe(Stkframe *frame, void *wbufp);
-static void addstackroots(G *gp, Workbuf **wbufp);
-
-static FuncVal runfinqv = {runfinq};
-static FuncVal bgsweepv = {bgsweep};
-
-static struct {
- uint64 full; // lock-free list of full blocks
- uint64 empty; // lock-free list of empty blocks
- byte pad0[CacheLineSize]; // prevents false-sharing between full/empty and nproc/nwait
- uint32 nproc;
- int64 tstart;
- volatile uint32 nwait;
- volatile uint32 ndone;
- Note alldone;
- ParFor *markfor;
-
- Lock;
- byte *chunk;
- uintptr nchunk;
-} work;
-
-enum {
- GC_DEFAULT_PTR = GC_NUM_INSTR,
- GC_CHAN,
-
- GC_NUM_INSTR2
-};
-
-static struct {
- struct {
- uint64 sum;
- uint64 cnt;
- } ptr;
- uint64 nbytes;
- struct {
- uint64 sum;
- uint64 cnt;
- uint64 notype;
- uint64 typelookup;
- } obj;
- uint64 rescan;
- uint64 rescanbytes;
- uint64 instr[GC_NUM_INSTR2];
- uint64 putempty;
- uint64 getfull;
- struct {
- uint64 foundbit;
- uint64 foundword;
- uint64 foundspan;
- } flushptrbuf;
- struct {
- uint64 foundbit;
- uint64 foundword;
- uint64 foundspan;
- } markonly;
- uint32 nbgsweep;
- uint32 npausesweep;
-} gcstats;
-
-// markonly marks an object. It returns true if the object
-// has been marked by this function, false otherwise.
-// This function doesn't append the object to any buffer.
-static bool
-markonly(void *obj)
-{
- byte *p;
- uintptr *bitp, bits, shift, x, xbits, off, j;
- MSpan *s;
- PageID k;
-
- // Words outside the arena cannot be pointers.
- if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
- return false;
-
- // obj may be a pointer to a live object.
- // Try to find the beginning of the object.
-
- // Round down to word boundary.
- obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
-
- // Find bits for this word.
- off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
- bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- xbits = *bitp;
- bits = xbits >> shift;
-
- // Pointing at the beginning of a block?
- if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
- if(CollectStats)
- runtime·xadd64(&gcstats.markonly.foundbit, 1);
- goto found;
- }
-
- // Pointing just past the beginning?
- // Scan backward a little to find a block boundary.
- for(j=shift; j-->0; ) {
- if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
- shift = j;
- bits = xbits>>shift;
- if(CollectStats)
- runtime·xadd64(&gcstats.markonly.foundword, 1);
- goto found;
- }
- }
-
- // Otherwise consult span table to find beginning.
- // (Manually inlined copy of MHeap_LookupMaybe.)
- k = (uintptr)obj>>PageShift;
- x = k;
- x -= (uintptr)runtime·mheap.arena_start>>PageShift;
- s = runtime·mheap.spans[x];
- if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
- return false;
- p = (byte*)((uintptr)s->start<<PageShift);
- if(s->sizeclass == 0) {
- obj = p;
- } else {
- uintptr size = s->elemsize;
- int32 i = ((byte*)obj - p)/size;
- obj = p+i*size;
- }
-
- // Now that we know the object header, reload bits.
- off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
- bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- xbits = *bitp;
- bits = xbits >> shift;
- if(CollectStats)
- runtime·xadd64(&gcstats.markonly.foundspan, 1);
-
-found:
- // Now we have bits, bitp, and shift correct for
- // obj pointing at the base of the object.
- // Only care about allocated and not marked.
- if((bits & (bitAllocated|bitMarked)) != bitAllocated)
- return false;
- if(work.nproc == 1)
- *bitp |= bitMarked<<shift;
- else {
- for(;;) {
- x = *bitp;
- if(x & (bitMarked<<shift))
- return false;
- if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
- break;
- }
- }
-
- // The object is now marked
- return true;
-}
-
-// PtrTarget is a structure used by intermediate buffers.
-// The intermediate buffers hold GC data before it
-// is moved/flushed to the work buffer (Workbuf).
-// The size of an intermediate buffer is very small,
-// such as 32 or 64 elements.
-typedef struct PtrTarget PtrTarget;
-struct PtrTarget
-{
- void *p;
- uintptr ti;
-};
-
-typedef struct Scanbuf Scanbuf;
-struct Scanbuf
-{
- struct {
- PtrTarget *begin;
- PtrTarget *end;
- PtrTarget *pos;
- } ptr;
- struct {
- Obj *begin;
- Obj *end;
- Obj *pos;
- } obj;
- Workbuf *wbuf;
- Obj *wp;
- uintptr nobj;
-};
-
-typedef struct BufferList BufferList;
-struct BufferList
-{
- PtrTarget ptrtarget[IntermediateBufferCapacity];
- Obj obj[IntermediateBufferCapacity];
- uint32 busy;
- byte pad[CacheLineSize];
-};
-#pragma dataflag NOPTR
-static BufferList bufferList[MaxGcproc];
-
-static Type *itabtype;
-
-static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
-
-// flushptrbuf moves data from the PtrTarget buffer to the work buffer.
-// The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
-// while the work buffer contains blocks which have been marked
-// and are prepared to be scanned by the garbage collector.
-//
-// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
-//
-// A simplified drawing explaining how the todo-list moves from a structure to another:
-//
-// scanblock
-// (find pointers)
-// Obj ------> PtrTarget (pointer targets)
-// ↑ |
-// | |
-// `----------'
-// flushptrbuf
-// (find block start, mark and enqueue)
-static void
-flushptrbuf(Scanbuf *sbuf)
-{
- byte *p, *arena_start, *obj;
- uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n;
- MSpan *s;
- PageID k;
- Obj *wp;
- Workbuf *wbuf;
- PtrTarget *ptrbuf;
- PtrTarget *ptrbuf_end;
-
- arena_start = runtime·mheap.arena_start;
-
- wp = sbuf->wp;
- wbuf = sbuf->wbuf;
- nobj = sbuf->nobj;
-
- ptrbuf = sbuf->ptr.begin;
- ptrbuf_end = sbuf->ptr.pos;
- n = ptrbuf_end - sbuf->ptr.begin;
- sbuf->ptr.pos = sbuf->ptr.begin;
-
- if(CollectStats) {
- runtime·xadd64(&gcstats.ptr.sum, n);
- runtime·xadd64(&gcstats.ptr.cnt, 1);
- }
-
- // If buffer is nearly full, get a new one.
- if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) {
- if(wbuf != nil)
- wbuf->nobj = nobj;
- wbuf = getempty(wbuf);
- wp = wbuf->obj;
- nobj = 0;
-
- if(n >= nelem(wbuf->obj))
- runtime·throw("ptrbuf has to be smaller than WorkBuf");
- }
-
- while(ptrbuf < ptrbuf_end) {
- obj = ptrbuf->p;
- ti = ptrbuf->ti;
- ptrbuf++;
-
- // obj belongs to interval [mheap.arena_start, mheap.arena_used).
- if(Debug > 1) {
- if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
- runtime·throw("object is outside of mheap");
- }
-
- // obj may be a pointer to a live object.
- // Try to find the beginning of the object.
-
- // Round down to word boundary.
- if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) {
- obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
- ti = 0;
- }
-
- // Find bits for this word.
- off = (uintptr*)obj - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- xbits = *bitp;
- bits = xbits >> shift;
-
- // Pointing at the beginning of a block?
- if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
- if(CollectStats)
- runtime·xadd64(&gcstats.flushptrbuf.foundbit, 1);
- goto found;
- }
-
- ti = 0;
-
- // Pointing just past the beginning?
- // Scan backward a little to find a block boundary.
- for(j=shift; j-->0; ) {
- if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
- obj = (byte*)obj - (shift-j)*PtrSize;
- shift = j;
- bits = xbits>>shift;
- if(CollectStats)
- runtime·xadd64(&gcstats.flushptrbuf.foundword, 1);
- goto found;
- }
- }
-
- // Otherwise consult span table to find beginning.
- // (Manually inlined copy of MHeap_LookupMaybe.)
- k = (uintptr)obj>>PageShift;
- x = k;
- x -= (uintptr)arena_start>>PageShift;
- s = runtime·mheap.spans[x];
- if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
- continue;
- p = (byte*)((uintptr)s->start<<PageShift);
- if(s->sizeclass == 0) {
- obj = p;
- } else {
- size = s->elemsize;
- int32 i = ((byte*)obj - p)/size;
- obj = p+i*size;
- }
-
- // Now that we know the object header, reload bits.
- off = (uintptr*)obj - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- xbits = *bitp;
- bits = xbits >> shift;
- if(CollectStats)
- runtime·xadd64(&gcstats.flushptrbuf.foundspan, 1);
-
- found:
- // Now we have bits, bitp, and shift correct for
- // obj pointing at the base of the object.
- // Only care about allocated and not marked.
- if((bits & (bitAllocated|bitMarked)) != bitAllocated)
- continue;
- if(work.nproc == 1)
- *bitp |= bitMarked<<shift;
- else {
- for(;;) {
- x = *bitp;
- if(x & (bitMarked<<shift))
- goto continue_obj;
- if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
- break;
- }
- }
-
- // If object has no pointers, don't need to scan further.
- if((bits & bitScan) == 0)
- continue;
-
- // Ask span about size class.
- // (Manually inlined copy of MHeap_Lookup.)
- x = (uintptr)obj >> PageShift;
- x -= (uintptr)arena_start>>PageShift;
- s = runtime·mheap.spans[x];
-
- PREFETCH(obj);
-
- *wp = (Obj){obj, s->elemsize, ti};
- wp++;
- nobj++;
- continue_obj:;
- }
-
- // If another proc wants a pointer, give it some.
- if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
- wbuf->nobj = nobj;
- wbuf = handoff(wbuf);
- nobj = wbuf->nobj;
- wp = wbuf->obj + nobj;
- }
-
- sbuf->wp = wp;
- sbuf->wbuf = wbuf;
- sbuf->nobj = nobj;
-}
-
-static void
-flushobjbuf(Scanbuf *sbuf)
-{
- uintptr nobj, off;
- Obj *wp, obj;
- Workbuf *wbuf;
- Obj *objbuf;
- Obj *objbuf_end;
-
- wp = sbuf->wp;
- wbuf = sbuf->wbuf;
- nobj = sbuf->nobj;
-
- objbuf = sbuf->obj.begin;
- objbuf_end = sbuf->obj.pos;
- sbuf->obj.pos = sbuf->obj.begin;
-
- while(objbuf < objbuf_end) {
- obj = *objbuf++;
-
- // Align obj.b to a word boundary.
- off = (uintptr)obj.p & (PtrSize-1);
- if(off != 0) {
- obj.p += PtrSize - off;
- obj.n -= PtrSize - off;
- obj.ti = 0;
- }
-
- if(obj.p == nil || obj.n == 0)
- continue;
-
- // If buffer is full, get a new one.
- if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
- if(wbuf != nil)
- wbuf->nobj = nobj;
- wbuf = getempty(wbuf);
- wp = wbuf->obj;
- nobj = 0;
- }
-
- *wp = obj;
- wp++;
- nobj++;
- }
-
- // If another proc wants a pointer, give it some.
- if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
- wbuf->nobj = nobj;
- wbuf = handoff(wbuf);
- nobj = wbuf->nobj;
- wp = wbuf->obj + nobj;
- }
-
- sbuf->wp = wp;
- sbuf->wbuf = wbuf;
- sbuf->nobj = nobj;
-}
-
-// Program that scans the whole block and treats every block element as a potential pointer
-static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
-
-// Hchan program
-static uintptr chanProg[2] = {0, GC_CHAN};
-
-// Local variables of a program fragment or loop
-typedef struct Frame Frame;
-struct Frame {
- uintptr count, elemsize, b;
- uintptr *loop_or_ret;
-};
-
-// Sanity check for the derived type info objti.
-static void
-checkptr(void *obj, uintptr objti)
-{
- uintptr *pc1, *pc2, type, tisize, i, j, x;
- byte *objstart;
- Type *t;
- MSpan *s;
-
- if(!Debug)
- runtime·throw("checkptr is debug only");
-
- if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
- return;
- type = runtime·gettype(obj);
- t = (Type*)(type & ~(uintptr)(PtrSize-1));
- if(t == nil)
- return;
- x = (uintptr)obj >> PageShift;
- x -= (uintptr)(runtime·mheap.arena_start)>>PageShift;
- s = runtime·mheap.spans[x];
- objstart = (byte*)((uintptr)s->start<<PageShift);
- if(s->sizeclass != 0) {
- i = ((byte*)obj - objstart)/s->elemsize;
- objstart += i*s->elemsize;
- }
- tisize = *(uintptr*)objti;
- // Sanity check for object size: it should fit into the memory block.
- if((byte*)obj + tisize > objstart + s->elemsize) {
- runtime·printf("object of type '%S' at %p/%p does not fit in block %p/%p\n",
- *t->string, obj, tisize, objstart, s->elemsize);
- runtime·throw("invalid gc type info");
- }
- if(obj != objstart)
- return;
- // If obj points to the beginning of the memory block,
- // check type info as well.
- if(t->string == nil ||
- // Gob allocates unsafe pointers for indirection.
- (runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") &&
- // Runtime and gc think differently about closures.
- runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t->string->str)) {
- pc1 = (uintptr*)objti;
- pc2 = (uintptr*)t->gc;
- // A simple best-effort check until first GC_END.
- for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) {
- if(pc1[j] != pc2[j]) {
- runtime·printf("invalid gc type info for '%s', type info %p [%d]=%p, block info %p [%d]=%p\n",
- t->string ? (int8*)t->string->str : (int8*)"?", pc1, (int32)j, pc1[j], pc2, (int32)j, pc2[j]);
- runtime·throw("invalid gc type info");
- }
- }
- }
-}
-
-// scanblock scans a block of n bytes starting at pointer b for references
-// to other objects, scanning any it finds recursively until there are no
-// unscanned objects left. Instead of using an explicit recursion, it keeps
-// a work list in the Workbuf* structures and loops in the main function
-// body. Keeping an explicit work list is easier on the stack allocator and
-// more efficient.
-static void
-scanblock(Workbuf *wbuf, bool keepworking)
-{
- byte *b, *arena_start, *arena_used;
- uintptr n, i, end_b, elemsize, size, ti, objti, count, type, nobj;
- uintptr *pc, precise_type, nominal_size;
- uintptr *chan_ret, chancap;
- void *obj;
- Type *t, *et;
- Slice *sliceptr;
- String *stringptr;
- Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
- BufferList *scanbuffers;
- Scanbuf sbuf;
- Eface *eface;
- Iface *iface;
- Hchan *chan;
- ChanType *chantype;
- Obj *wp;
-
- if(sizeof(Workbuf) % WorkbufSize != 0)
- runtime·throw("scanblock: size of Workbuf is suboptimal");
-
- // Memory arena parameters.
- arena_start = runtime·mheap.arena_start;
- arena_used = runtime·mheap.arena_used;
-
- stack_ptr = stack+nelem(stack)-1;
-
- precise_type = false;
- nominal_size = 0;
-
- if(wbuf) {
- nobj = wbuf->nobj;
- wp = &wbuf->obj[nobj];
- } else {
- nobj = 0;
- wp = nil;
- }
-
- // Initialize sbuf
- scanbuffers = &bufferList[m->helpgc];
-
- sbuf.ptr.begin = sbuf.ptr.pos = &scanbuffers->ptrtarget[0];
- sbuf.ptr.end = sbuf.ptr.begin + nelem(scanbuffers->ptrtarget);
-
- sbuf.obj.begin = sbuf.obj.pos = &scanbuffers->obj[0];
- sbuf.obj.end = sbuf.obj.begin + nelem(scanbuffers->obj);
-
- sbuf.wbuf = wbuf;
- sbuf.wp = wp;
- sbuf.nobj = nobj;
-
- // (Silence the compiler)
- chan = nil;
- chantype = nil;
- chan_ret = nil;
-
- goto next_block;
-
- for(;;) {
- // Each iteration scans the block b of length n, queueing pointers in
- // the work buffer.
-
- if(CollectStats) {
- runtime·xadd64(&gcstats.nbytes, n);
- runtime·xadd64(&gcstats.obj.sum, sbuf.nobj);
- runtime·xadd64(&gcstats.obj.cnt, 1);
- }
-
- if(ti != 0) {
- if(Debug > 1) {
- runtime·printf("scanblock %p %D ti %p\n", b, (int64)n, ti);
- }
- pc = (uintptr*)(ti & ~(uintptr)PC_BITS);
- precise_type = (ti & PRECISE);
- stack_top.elemsize = pc[0];
- if(!precise_type)
- nominal_size = pc[0];
- if(ti & LOOP) {
- stack_top.count = 0; // 0 means an infinite number of iterations
- stack_top.loop_or_ret = pc+1;
- } else {
- stack_top.count = 1;
- }
- if(Debug) {
- // Simple sanity check for provided type info ti:
- // The declared size of the object must be not larger than the actual size
- // (it can be smaller due to inferior pointers).
- // It's difficult to make a comprehensive check due to inferior pointers,
- // reflection, gob, etc.
- if(pc[0] > n) {
- runtime·printf("invalid gc type info: type info size %p, block size %p\n", pc[0], n);
- runtime·throw("invalid gc type info");
- }
- }
- } else if(UseSpanType) {
- if(CollectStats)
- runtime·xadd64(&gcstats.obj.notype, 1);
-
- type = runtime·gettype(b);
- if(type != 0) {
- if(CollectStats)
- runtime·xadd64(&gcstats.obj.typelookup, 1);
-
- t = (Type*)(type & ~(uintptr)(PtrSize-1));
- switch(type & (PtrSize-1)) {
- case TypeInfo_SingleObject:
- pc = (uintptr*)t->gc;
- precise_type = true; // type information about 'b' is precise
- stack_top.count = 1;
- stack_top.elemsize = pc[0];
- break;
- case TypeInfo_Array:
- pc = (uintptr*)t->gc;
- if(pc[0] == 0)
- goto next_block;
- precise_type = true; // type information about 'b' is precise
- stack_top.count = 0; // 0 means an infinite number of iterations
- stack_top.elemsize = pc[0];
- stack_top.loop_or_ret = pc+1;
- break;
- case TypeInfo_Chan:
- chan = (Hchan*)b;
- chantype = (ChanType*)t;
- chan_ret = nil;
- pc = chanProg;
- break;
- default:
- if(Debug > 1)
- runtime·printf("scanblock %p %D type %p %S\n", b, (int64)n, type, *t->string);
- runtime·throw("scanblock: invalid type");
- return;
- }
- if(Debug > 1)
- runtime·printf("scanblock %p %D type %p %S pc=%p\n", b, (int64)n, type, *t->string, pc);
- } else {
- pc = defaultProg;
- if(Debug > 1)
- runtime·printf("scanblock %p %D unknown type\n", b, (int64)n);
- }
- } else {
- pc = defaultProg;
- if(Debug > 1)
- runtime·printf("scanblock %p %D no span types\n", b, (int64)n);
- }
-
- if(IgnorePreciseGC)
- pc = defaultProg;
-
- pc++;
- stack_top.b = (uintptr)b;
- end_b = (uintptr)b + n - PtrSize;
-
- for(;;) {
- if(CollectStats)
- runtime·xadd64(&gcstats.instr[pc[0]], 1);
-
- obj = nil;
- objti = 0;
- switch(pc[0]) {
- case GC_PTR:
- obj = *(void**)(stack_top.b + pc[1]);
- objti = pc[2];
- if(Debug > 2)
- runtime·printf("gc_ptr @%p: %p ti=%p\n", stack_top.b+pc[1], obj, objti);
- pc += 3;
- if(Debug)
- checkptr(obj, objti);
- break;
-
- case GC_SLICE:
- sliceptr = (Slice*)(stack_top.b + pc[1]);
- if(Debug > 2)
- runtime·printf("gc_slice @%p: %p/%D/%D\n", sliceptr, sliceptr->array, (int64)sliceptr->len, (int64)sliceptr->cap);
- if(sliceptr->cap != 0) {
- obj = sliceptr->array;
- // Can't use slice element type for scanning,
- // because if it points to an array embedded
- // in the beginning of a struct,
- // we will scan the whole struct as the slice.
- // So just obtain type info from heap.
- }
- pc += 3;
- break;
-
- case GC_APTR:
- obj = *(void**)(stack_top.b + pc[1]);
- if(Debug > 2)
- runtime·printf("gc_aptr @%p: %p\n", stack_top.b+pc[1], obj);
- pc += 2;
- break;
-
- case GC_STRING:
- stringptr = (String*)(stack_top.b + pc[1]);
- if(Debug > 2)
- runtime·printf("gc_string @%p: %p/%D\n", stack_top.b+pc[1], stringptr->str, (int64)stringptr->len);
- if(stringptr->len != 0)
- markonly(stringptr->str);
- pc += 2;
- continue;
-
- case GC_EFACE:
- eface = (Eface*)(stack_top.b + pc[1]);
- pc += 2;
- if(Debug > 2)
- runtime·printf("gc_eface @%p: %p %p\n", stack_top.b+pc[1], eface->type, eface->data);
- if(eface->type == nil)
- continue;
-
- // eface->type
- t = eface->type;
- if((void*)t >= arena_start && (void*)t < arena_used) {
- *sbuf.ptr.pos++ = (PtrTarget){t, 0};
- if(sbuf.ptr.pos == sbuf.ptr.end)
- flushptrbuf(&sbuf);
- }
-
- // eface->data
- if(eface->data >= arena_start && eface->data < arena_used) {
- if(t->size <= sizeof(void*)) {
- if((t->kind & KindNoPointers))
- continue;
-
- obj = eface->data;
- if((t->kind & ~KindNoPointers) == KindPtr) {
- // Only use type information if it is a pointer-containing type.
- // This matches the GC programs written by cmd/gc/reflect.c's
- // dgcsym1 in case TPTR32/case TPTR64. See rationale there.
- et = ((PtrType*)t)->elem;
- if(!(et->kind & KindNoPointers))
- objti = (uintptr)((PtrType*)t)->elem->gc;
- }
- } else {
- obj = eface->data;
- objti = (uintptr)t->gc;
- }
- }
- break;
-
- case GC_IFACE:
- iface = (Iface*)(stack_top.b + pc[1]);
- pc += 2;
- if(Debug > 2)
- runtime·printf("gc_iface @%p: %p/%p %p\n", stack_top.b+pc[1], iface->tab, nil, iface->data);
- if(iface->tab == nil)
- continue;
-
- // iface->tab
- if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) {
- *sbuf.ptr.pos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc};
- if(sbuf.ptr.pos == sbuf.ptr.end)
- flushptrbuf(&sbuf);
- }
-
- // iface->data
- if(iface->data >= arena_start && iface->data < arena_used) {
- t = iface->tab->type;
- if(t->size <= sizeof(void*)) {
- if((t->kind & KindNoPointers))
- continue;
-
- obj = iface->data;
- if((t->kind & ~KindNoPointers) == KindPtr) {
- // Only use type information if it is a pointer-containing type.
- // This matches the GC programs written by cmd/gc/reflect.c's
- // dgcsym1 in case TPTR32/case TPTR64. See rationale there.
- et = ((PtrType*)t)->elem;
- if(!(et->kind & KindNoPointers))
- objti = (uintptr)((PtrType*)t)->elem->gc;
- }
- } else {
- obj = iface->data;
- objti = (uintptr)t->gc;
- }
- }
- break;
-
- case GC_DEFAULT_PTR:
- while(stack_top.b <= end_b) {
- obj = *(byte**)stack_top.b;
- if(Debug > 2)
- runtime·printf("gc_default_ptr @%p: %p\n", stack_top.b, obj);
- stack_top.b += PtrSize;
- if(obj >= arena_start && obj < arena_used) {
- *sbuf.ptr.pos++ = (PtrTarget){obj, 0};
- if(sbuf.ptr.pos == sbuf.ptr.end)
- flushptrbuf(&sbuf);
- }
- }
- goto next_block;
-
- case GC_END:
- if(--stack_top.count != 0) {
- // Next iteration of a loop if possible.
- stack_top.b += stack_top.elemsize;
- if(stack_top.b + stack_top.elemsize <= end_b+PtrSize) {
- pc = stack_top.loop_or_ret;
- continue;
- }
- i = stack_top.b;
- } else {
- // Stack pop if possible.
- if(stack_ptr+1 < stack+nelem(stack)) {
- pc = stack_top.loop_or_ret;
- stack_top = *(++stack_ptr);
- continue;
- }
- i = (uintptr)b + nominal_size;
- }
- if(!precise_type) {
- // Quickly scan [b+i,b+n) for possible pointers.
- for(; i<=end_b; i+=PtrSize) {
- if(*(byte**)i != nil) {
- // Found a value that may be a pointer.
- // Do a rescan of the entire block.
- enqueue((Obj){b, n, 0}, &sbuf.wbuf, &sbuf.wp, &sbuf.nobj);
- if(CollectStats) {
- runtime·xadd64(&gcstats.rescan, 1);
- runtime·xadd64(&gcstats.rescanbytes, n);
- }
- break;
- }
- }
- }
- goto next_block;
-
- case GC_ARRAY_START:
- i = stack_top.b + pc[1];
- count = pc[2];
- elemsize = pc[3];
- pc += 4;
-
- // Stack push.
- *stack_ptr-- = stack_top;
- stack_top = (Frame){count, elemsize, i, pc};
- continue;
-
- case GC_ARRAY_NEXT:
- if(--stack_top.count != 0) {
- stack_top.b += stack_top.elemsize;
- pc = stack_top.loop_or_ret;
- } else {
- // Stack pop.
- stack_top = *(++stack_ptr);
- pc += 1;
- }
- continue;
-
- case GC_CALL:
- // Stack push.
- *stack_ptr-- = stack_top;
- stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*return address*/};
- pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target of the CALL instruction
- continue;
-
- case GC_REGION:
- obj = (void*)(stack_top.b + pc[1]);
- size = pc[2];
- objti = pc[3];
- pc += 4;
-
- if(Debug > 2)
- runtime·printf("gc_region @%p: %D %p\n", stack_top.b+pc[1], (int64)size, objti);
- *sbuf.obj.pos++ = (Obj){obj, size, objti};
- if(sbuf.obj.pos == sbuf.obj.end)
- flushobjbuf(&sbuf);
- continue;
-
- case GC_CHAN_PTR:
- chan = *(Hchan**)(stack_top.b + pc[1]);
- if(Debug > 2 && chan != nil)
- runtime·printf("gc_chan_ptr @%p: %p/%D/%D %p\n", stack_top.b+pc[1], chan, (int64)chan->qcount, (int64)chan->dataqsiz, pc[2]);
- if(chan == nil) {
- pc += 3;
- continue;
- }
- if(markonly(chan)) {
- chantype = (ChanType*)pc[2];
- if(!(chantype->elem->kind & KindNoPointers)) {
- // Start chanProg.
- chan_ret = pc+3;
- pc = chanProg+1;
- continue;
- }
- }
- pc += 3;
- continue;
-
- case GC_CHAN:
- // There are no heap pointers in struct Hchan,
- // so we can ignore the leading sizeof(Hchan) bytes.
- if(!(chantype->elem->kind & KindNoPointers)) {
- // Channel's buffer follows Hchan immediately in memory.
- // Size of buffer (cap(c)) is second int in the chan struct.
- chancap = ((uintgo*)chan)[1];
- if(chancap > 0) {
- // TODO(atom): split into two chunks so that only the
- // in-use part of the circular buffer is scanned.
- // (Channel routines zero the unused part, so the current
- // code does not lead to leaks, it's just a little inefficient.)
- *sbuf.obj.pos++ = (Obj){(byte*)chan+runtime·Hchansize, chancap*chantype->elem->size,
- (uintptr)chantype->elem->gc | PRECISE | LOOP};
- if(sbuf.obj.pos == sbuf.obj.end)
- flushobjbuf(&sbuf);
- }
- }
- if(chan_ret == nil)
- goto next_block;
- pc = chan_ret;
- continue;
-
- default:
- runtime·printf("runtime: invalid GC instruction %p at %p\n", pc[0], pc);
- runtime·throw("scanblock: invalid GC instruction");
- return;
- }
-
- if(obj >= arena_start && obj < arena_used) {
- *sbuf.ptr.pos++ = (PtrTarget){obj, objti};
- if(sbuf.ptr.pos == sbuf.ptr.end)
- flushptrbuf(&sbuf);
- }
- }
-
- next_block:
- // Done scanning [b, b+n). Prepare for the next iteration of
- // the loop by setting b, n, ti to the parameters for the next block.
-
- if(sbuf.nobj == 0) {
- flushptrbuf(&sbuf);
- flushobjbuf(&sbuf);
-
- if(sbuf.nobj == 0) {
- if(!keepworking) {
- if(sbuf.wbuf)
- putempty(sbuf.wbuf);
- return;
- }
- // Emptied our buffer: refill.
- sbuf.wbuf = getfull(sbuf.wbuf);
- if(sbuf.wbuf == nil)
- return;
- sbuf.nobj = sbuf.wbuf->nobj;
- sbuf.wp = sbuf.wbuf->obj + sbuf.wbuf->nobj;
- }
- }
-
- // Fetch b from the work buffer.
- --sbuf.wp;
- b = sbuf.wp->p;
- n = sbuf.wp->n;
- ti = sbuf.wp->ti;
- sbuf.nobj--;
- }
-}
-
-// Append obj to the work buffer.
-// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
-static void
-enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj)
-{
- uintptr nobj, off;
- Obj *wp;
- Workbuf *wbuf;
-
- if(Debug > 1)
- runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, obj.ti);
-
- // Align obj.b to a word boundary.
- off = (uintptr)obj.p & (PtrSize-1);
- if(off != 0) {
- obj.p += PtrSize - off;
- obj.n -= PtrSize - off;
- obj.ti = 0;
- }
-
- if(obj.p == nil || obj.n == 0)
- return;
-
- // Load work buffer state
- wp = *_wp;
- wbuf = *_wbuf;
- nobj = *_nobj;
-
- // If another proc wants a pointer, give it some.
- if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
- wbuf->nobj = nobj;
- wbuf = handoff(wbuf);
- nobj = wbuf->nobj;
- wp = wbuf->obj + nobj;
- }
-
- // If buffer is full, get a new one.
- if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
- if(wbuf != nil)
- wbuf->nobj = nobj;
- wbuf = getempty(wbuf);
- wp = wbuf->obj;
- nobj = 0;
- }
-
- *wp = obj;
- wp++;
- nobj++;
-
- // Save work buffer state
- *_wp = wp;
- *_wbuf = wbuf;
- *_nobj = nobj;
-}
-
-static void
-enqueue1(Workbuf **wbufp, Obj obj)
-{
- Workbuf *wbuf;
-
- wbuf = *wbufp;
- if(wbuf->nobj >= nelem(wbuf->obj))
- *wbufp = wbuf = getempty(wbuf);
- wbuf->obj[wbuf->nobj++] = obj;
-}
-
-static void
-markroot(ParFor *desc, uint32 i)
-{
- Workbuf *wbuf;
- FinBlock *fb;
- MHeap *h;
- MSpan **allspans, *s;
- uint32 spanidx, sg;
- G *gp;
- void *p;
-
- USED(&desc);
- wbuf = getempty(nil);
- // Note: if you add a case here, please also update heapdump.c:dumproots.
- switch(i) {
- case RootData:
- enqueue1(&wbuf, (Obj){data, edata - data, (uintptr)gcdata});
- break;
-
- case RootBss:
- enqueue1(&wbuf, (Obj){bss, ebss - bss, (uintptr)gcbss});
- break;
-
- case RootFinalizers:
- for(fb=allfin; fb; fb=fb->alllink)
- enqueue1(&wbuf, (Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
- break;
-
- case RootSpanTypes:
- // mark span types and MSpan.specials (to walk spans only once)
- h = &runtime·mheap;
- sg = h->sweepgen;
- allspans = h->allspans;
- for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
- Special *sp;
- SpecialFinalizer *spf;
-
- s = allspans[spanidx];
- if(s->sweepgen != sg) {
- runtime·printf("sweep %d %d\n", s->sweepgen, sg);
- runtime·throw("gc: unswept span");
- }
- if(s->state != MSpanInUse)
- continue;
- // The garbage collector ignores type pointers stored in MSpan.types:
- // - Compiler-generated types are stored outside of heap.
- // - The reflect package has runtime-generated types cached in its data structures.
- // The garbage collector relies on finding the references via that cache.
- if(s->types.compression == MTypes_Words || s->types.compression == MTypes_Bytes)
- markonly((byte*)s->types.data);
- for(sp = s->specials; sp != nil; sp = sp->next) {
- if(sp->kind != KindSpecialFinalizer)
- continue;
- // don't mark finalized object, but scan it so we
- // retain everything it points to.
- spf = (SpecialFinalizer*)sp;
- // A finalizer can be set for an inner byte of an object, find object beginning.
- p = (void*)((s->start << PageShift) + spf->offset/s->elemsize*s->elemsize);
- enqueue1(&wbuf, (Obj){p, s->elemsize, 0});
- enqueue1(&wbuf, (Obj){(void*)&spf->fn, PtrSize, 0});
- enqueue1(&wbuf, (Obj){(void*)&spf->fint, PtrSize, 0});
- enqueue1(&wbuf, (Obj){(void*)&spf->ot, PtrSize, 0});
- }
- }
- break;
-
- case RootFlushCaches:
- flushallmcaches();
- break;
-
- default:
- // the rest is scanning goroutine stacks
- if(i - RootCount >= runtime·allglen)
- runtime·throw("markroot: bad index");
- gp = runtime·allg[i - RootCount];
- // remember when we've first observed the G blocked
- // needed only to output in traceback
- if((gp->status == Gwaiting || gp->status == Gsyscall) && gp->waitsince == 0)
- gp->waitsince = work.tstart;
- addstackroots(gp, &wbuf);
- break;
-
- }
-
- if(wbuf)
- scanblock(wbuf, false);
-}
-
-// Get an empty work buffer off the work.empty list,
-// allocating new buffers as needed.
-static Workbuf*
-getempty(Workbuf *b)
-{
- if(b != nil)
- runtime·lfstackpush(&work.full, &b->node);
- b = (Workbuf*)runtime·lfstackpop(&work.empty);
- if(b == nil) {
- // Need to allocate.
- runtime·lock(&work);
- if(work.nchunk < sizeof *b) {
- work.nchunk = 1<<20;
- work.chunk = runtime·SysAlloc(work.nchunk, &mstats.gc_sys);
- if(work.chunk == nil)
- runtime·throw("runtime: cannot allocate memory");
- }
- b = (Workbuf*)work.chunk;
- work.chunk += sizeof *b;
- work.nchunk -= sizeof *b;
- runtime·unlock(&work);
- }
- b->nobj = 0;
- return b;
-}
-
-static void
-putempty(Workbuf *b)
-{
- if(CollectStats)
- runtime·xadd64(&gcstats.putempty, 1);
-
- runtime·lfstackpush(&work.empty, &b->node);
-}
-
-// Get a full work buffer off the work.full list, or return nil.
-static Workbuf*
-getfull(Workbuf *b)
-{
- int32 i;
-
- if(CollectStats)
- runtime·xadd64(&gcstats.getfull, 1);
-
- if(b != nil)
- runtime·lfstackpush(&work.empty, &b->node);
- b = (Workbuf*)runtime·lfstackpop(&work.full);
- if(b != nil || work.nproc == 1)
- return b;
-
- runtime·xadd(&work.nwait, +1);
- for(i=0;; i++) {
- if(work.full != 0) {
- runtime·xadd(&work.nwait, -1);
- b = (Workbuf*)runtime·lfstackpop(&work.full);
- if(b != nil)
- return b;
- runtime·xadd(&work.nwait, +1);
- }
- if(work.nwait == work.nproc)
- return nil;
- if(i < 10) {
- m->gcstats.nprocyield++;
- runtime·procyield(20);
- } else if(i < 20) {
- m->gcstats.nosyield++;
- runtime·osyield();
- } else {
- m->gcstats.nsleep++;
- runtime·usleep(100);
- }
- }
-}
-
-static Workbuf*
-handoff(Workbuf *b)
-{
- int32 n;
- Workbuf *b1;
-
- // Make new buffer with half of b's pointers.
- b1 = getempty(nil);
- n = b->nobj/2;
- b->nobj -= n;
- b1->nobj = n;
- runtime·memmove(b1->obj, b->obj+b->nobj, n*sizeof b1->obj[0]);
- m->gcstats.nhandoff++;
- m->gcstats.nhandoffcnt += n;
-
- // Put b on full list - let first half of b get stolen.
- runtime·lfstackpush(&work.full, &b->node);
- return b1;
-}
-
-extern byte pclntab[]; // base for f->ptrsoff
-
-BitVector
-runtime·stackmapdata(StackMap *stackmap, int32 n)
-{
- if(n < 0 || n >= stackmap->n)
- runtime·throw("stackmapdata: index out of range");
- return (BitVector){stackmap->nbit, stackmap->data + n*((stackmap->nbit+31)/32)};
-}
-
-// Scans an interface data value when the interface type indicates
-// that it is a pointer.
-static void
-scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue, void *wbufp)
-{
- Itab *tab;
- Type *type;
-
- if(runtime·precisestack && afterprologue) {
- if(bits == BitsIface) {
- tab = *(Itab**)scanp;
- if(tab->type->size <= sizeof(void*) && (tab->type->kind & KindNoPointers))
- return;
- } else { // bits == BitsEface
- type = *(Type**)scanp;
- if(type->size <= sizeof(void*) && (type->kind & KindNoPointers))
- return;
- }
- }
- enqueue1(wbufp, (Obj){scanp+PtrSize, PtrSize, 0});
-}
-
-// Starting from scanp, scans words corresponding to set bits.
-static void
-scanbitvector(Func *f, bool precise, byte *scanp, BitVector *bv, bool afterprologue, void *wbufp)
-{
- uintptr word, bits;
- uint32 *wordp;
- int32 i, remptrs;
- byte *p;
-
- wordp = bv->data;
- for(remptrs = bv->n; remptrs > 0; remptrs -= 32) {
- word = *wordp++;
- if(remptrs < 32)
- i = remptrs;
- else
- i = 32;
- i /= BitsPerPointer;
- for(; i > 0; i--) {
- bits = word & 3;
- switch(bits) {
- case BitsDead:
- if(runtime·debug.gcdead)
- *(uintptr*)scanp = PoisonGC;
- break;
- case BitsScalar:
- break;
- case BitsPointer:
- p = *(byte**)scanp;
- if(p != nil) {
- if(Debug > 2)
- runtime·printf("frame %s @%p: ptr %p\n", runtime·funcname(f), scanp, p);
- if(precise && (p < (byte*)PageSize || (uintptr)p == PoisonGC || (uintptr)p == PoisonStack)) {
- // Looks like a junk value in a pointer slot.
- // Liveness analysis wrong?
- m->traceback = 2;
- runtime·printf("bad pointer in frame %s at %p: %p\n", runtime·funcname(f), scanp, p);
- runtime·throw("bad pointer in scanbitvector");
- }
- enqueue1(wbufp, (Obj){scanp, PtrSize, 0});
- }
- break;
- case BitsMultiWord:
- p = scanp;
- word >>= BitsPerPointer;
- scanp += PtrSize;
- i--;
- if(i == 0) {
- // Get next chunk of bits
- remptrs -= 32;
- word = *wordp++;
- if(remptrs < 32)
- i = remptrs;
- else
- i = 32;
- i /= BitsPerPointer;
- }
- switch(word & 3) {
- case BitsString:
- if(Debug > 2)
- runtime·printf("frame %s @%p: string %p/%D\n", runtime·funcname(f), p, ((String*)p)->str, (int64)((String*)p)->len);
- if(((String*)p)->len != 0)
- markonly(((String*)p)->str);
- break;
- case BitsSlice:
- word >>= BitsPerPointer;
- scanp += PtrSize;
- i--;
- if(i == 0) {
- // Get next chunk of bits
- remptrs -= 32;
- word = *wordp++;
- if(remptrs < 32)
- i = remptrs;
- else
- i = 32;
- i /= BitsPerPointer;
- }
- if(Debug > 2)
- runtime·printf("frame %s @%p: slice %p/%D/%D\n", runtime·funcname(f), p, ((Slice*)p)->array, (int64)((Slice*)p)->len, (int64)((Slice*)p)->cap);
- if(((Slice*)p)->cap < ((Slice*)p)->len) {
- m->traceback = 2;
- runtime·printf("bad slice in frame %s at %p: %p/%p/%p\n", runtime·funcname(f), p, ((byte**)p)[0], ((byte**)p)[1], ((byte**)p)[2]);
- runtime·throw("slice capacity smaller than length");
- }
- if(((Slice*)p)->cap != 0)
- enqueue1(wbufp, (Obj){p, PtrSize, 0});
- break;
- case BitsIface:
- case BitsEface:
- if(*(byte**)p != nil) {
- if(Debug > 2) {
- if((word&3) == BitsEface)
- runtime·printf("frame %s @%p: eface %p %p\n", runtime·funcname(f), p, ((uintptr*)p)[0], ((uintptr*)p)[1]);
- else
- runtime·printf("frame %s @%p: iface %p %p\n", runtime·funcname(f), p, ((uintptr*)p)[0], ((uintptr*)p)[1]);
- }
- scaninterfacedata(word & 3, p, afterprologue, wbufp);
- }
- break;
- }
- }
- word >>= BitsPerPointer;
- scanp += PtrSize;
- }
- }
-}
-
-// Scan a stack frame: local variables and function arguments/results.
-static bool
-scanframe(Stkframe *frame, void *wbufp)
-{
- Func *f;
- StackMap *stackmap;
- BitVector bv;
- uintptr size;
- uintptr targetpc;
- int32 pcdata;
- bool afterprologue;
- bool precise;
-
- f = frame->fn;
- targetpc = frame->continpc;
- if(targetpc == 0) {
- // Frame is dead.
- return true;
- }
- if(targetpc != f->entry)
- targetpc--;
- pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc);
- if(pcdata == -1) {
- // We do not have a valid pcdata value but there might be a
- // stackmap for this function. It is likely that we are looking
- // at the function prologue, assume so and hope for the best.
- pcdata = 0;
- }
-
- // Scan local variables if stack frame has been allocated.
- // Use pointer information if known.
- afterprologue = (frame->varp > (byte*)frame->sp);
- precise = false;
- if(afterprologue) {
- stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
- if(stackmap == nil) {
- // No locals information, scan everything.
- size = frame->varp - (byte*)frame->sp;
- if(Debug > 2)
- runtime·printf("frame %s unsized locals %p+%p\n", runtime·funcname(f), frame->varp-size, size);
- enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
- } else if(stackmap->n < 0) {
- // Locals size information, scan just the locals.
- size = -stackmap->n;
- if(Debug > 2)
- runtime·printf("frame %s conservative locals %p+%p\n", runtime·funcname(f), frame->varp-size, size);
- enqueue1(wbufp, (Obj){frame->varp - size, size, 0});
- } else if(stackmap->n > 0) {
- // Locals bitmap information, scan just the pointers in
- // locals.
- if(pcdata < 0 || pcdata >= stackmap->n) {
- // don't know where we are
- runtime·printf("pcdata is %d and %d stack map entries for %s (targetpc=%p)\n",
- pcdata, stackmap->n, runtime·funcname(f), targetpc);
- runtime·throw("scanframe: bad symbol table");
- }
- bv = runtime·stackmapdata(stackmap, pcdata);
- size = (bv.n * PtrSize) / BitsPerPointer;
- precise = true;
- scanbitvector(f, true, frame->varp - size, &bv, afterprologue, wbufp);
- }
- }
-
- // Scan arguments.
- // Use pointer information if known.
- stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
- if(stackmap != nil) {
- bv = runtime·stackmapdata(stackmap, pcdata);
- scanbitvector(f, precise, frame->argp, &bv, true, wbufp);
- } else {
- if(Debug > 2)
- runtime·printf("frame %s conservative args %p+%p\n", runtime·funcname(f), frame->argp, (uintptr)frame->arglen);
- enqueue1(wbufp, (Obj){frame->argp, frame->arglen, 0});
- }
- return true;
-}
-
-static void
-addstackroots(G *gp, Workbuf **wbufp)
-{
- M *mp;
- int32 n;
- Stktop *stk;
- uintptr sp, guard;
- void *base;
- uintptr size;
-
- switch(gp->status){
- default:
- runtime·printf("unexpected G.status %d (goroutine %p %D)\n", gp->status, gp, gp->goid);
- runtime·throw("mark - bad status");
- case Gdead:
- return;
- case Grunning:
- runtime·throw("mark - world not stopped");
- case Grunnable:
- case Gsyscall:
- case Gwaiting:
- break;
- }
-
- if(gp == g)
- runtime·throw("can't scan our own stack");
- if((mp = gp->m) != nil && mp->helpgc)
- runtime·throw("can't scan gchelper stack");
-
- if(gp->syscallstack != (uintptr)nil) {
- // Scanning another goroutine that is about to enter or might
- // have just exited a system call. It may be executing code such
- // as schedlock and may have needed to start a new stack segment.
- // Use the stack segment and stack pointer at the time of
- // the system call instead, since that won't change underfoot.
- sp = gp->syscallsp;
- stk = (Stktop*)gp->syscallstack;
- guard = gp->syscallguard;
- } else {
- // Scanning another goroutine's stack.
- // The goroutine is usually asleep (the world is stopped).
- sp = gp->sched.sp;
- stk = (Stktop*)gp->stackbase;
- guard = gp->stackguard;
- // For function about to start, context argument is a root too.
- if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
- enqueue1(wbufp, (Obj){base, size, 0});
- }
- if(ScanStackByFrames) {
- USED(sp);
- USED(stk);
- USED(guard);
- runtime·gentraceback(~(uintptr)0, ~(uintptr)0, 0, gp, 0, nil, 0x7fffffff, scanframe, wbufp, false);
- } else {
- n = 0;
- while(stk) {
- if(sp < guard-StackGuard || (uintptr)stk < sp) {
- runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
- runtime·throw("scanstack");
- }
- if(Debug > 2)
- runtime·printf("conservative stack %p+%p\n", (byte*)sp, (uintptr)stk-sp);
- enqueue1(wbufp, (Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
- sp = stk->gobuf.sp;
- guard = stk->stackguard;
- stk = (Stktop*)stk->stackbase;
- n++;
- }
- }
-}
-
-void
-runtime·queuefinalizer(byte *p, FuncVal *fn, uintptr nret, Type *fint, PtrType *ot)
-{
- FinBlock *block;
- Finalizer *f;
-
- runtime·lock(&finlock);
- if(finq == nil || finq->cnt == finq->cap) {
- if(finc == nil) {
- finc = runtime·persistentalloc(FinBlockSize, 0, &mstats.gc_sys);
- finc->cap = (FinBlockSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1;
- finc->alllink = allfin;
- allfin = finc;
- }
- block = finc;
- finc = block->next;
- block->next = finq;
- finq = block;
- }
- f = &finq->fin[finq->cnt];
- finq->cnt++;
- f->fn = fn;
- f->nret = nret;
- f->fint = fint;
- f->ot = ot;
- f->arg = p;
- runtime·fingwake = true;
- runtime·unlock(&finlock);
-}
-
-void
-runtime·iterate_finq(void (*callback)(FuncVal*, byte*, uintptr, Type*, PtrType*))
-{
- FinBlock *fb;
- Finalizer *f;
- uintptr i;
-
- for(fb = allfin; fb; fb = fb->alllink) {
- for(i = 0; i < fb->cnt; i++) {
- f = &fb->fin[i];
- callback(f->fn, f->arg, f->nret, f->fint, f->ot);
- }
- }
-}
-
-void
-runtime·MSpan_EnsureSwept(MSpan *s)
-{
- uint32 sg;
-
- // Caller must disable preemption.
- // Otherwise when this function returns the span can become unswept again
- // (if GC is triggered on another goroutine).
- if(m->locks == 0 && m->mallocing == 0 && g != m->g0)
- runtime·throw("MSpan_EnsureSwept: m is not locked");
-
- sg = runtime·mheap.sweepgen;
- if(runtime·atomicload(&s->sweepgen) == sg)
- return;
- if(runtime·cas(&s->sweepgen, sg-2, sg-1)) {
- runtime·MSpan_Sweep(s);
- return;
- }
- // unfortunate condition, and we don't have efficient means to wait
- while(runtime·atomicload(&s->sweepgen) != sg)
- runtime·osyield();
-}
-
-// Sweep frees or collects finalizers for blocks not marked in the mark phase.
-// It clears the mark bits in preparation for the next GC round.
-// Returns true if the span was returned to heap.
-bool
-runtime·MSpan_Sweep(MSpan *s)
-{
- int32 cl, n, npages, nfree;
- uintptr size, off, *bitp, shift, bits;
- uint32 sweepgen;
- byte *p;
- MCache *c;
- byte *arena_start;
- MLink head, *end;
- byte *type_data;
- byte compression;
- uintptr type_data_inc;
- MLink *x;
- Special *special, **specialp, *y;
- bool res, sweepgenset;
-
- // It's critical that we enter this function with preemption disabled,
- // GC must not start while we are in the middle of this function.
- if(m->locks == 0 && m->mallocing == 0 && g != m->g0)
- runtime·throw("MSpan_Sweep: m is not locked");
- sweepgen = runtime·mheap.sweepgen;
- if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) {
- runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n",
- s->state, s->sweepgen, sweepgen);
- runtime·throw("MSpan_Sweep: bad span state");
- }
- arena_start = runtime·mheap.arena_start;
- cl = s->sizeclass;
- size = s->elemsize;
- if(cl == 0) {
- n = 1;
- } else {
- // Chunk full of small blocks.
- npages = runtime·class_to_allocnpages[cl];
- n = (npages << PageShift) / size;
- }
- res = false;
- nfree = 0;
- end = &head;
- c = m->mcache;
- sweepgenset = false;
-
- // mark any free objects in this span so we don't collect them
- for(x = s->freelist; x != nil; x = x->next) {
- // This is markonly(x) but faster because we don't need
- // atomic access and we're guaranteed to be pointing at
- // the head of a valid object.
- off = (uintptr*)x - (uintptr*)runtime·mheap.arena_start;
- bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- *bitp |= bitMarked<<shift;
- }
-
- // Unlink & free special records for any objects we're about to free.
- specialp = &s->specials;
- special = *specialp;
- while(special != nil) {
- // A finalizer can be set for an inner byte of an object, find object beginning.
- p = (byte*)(s->start << PageShift) + special->offset/size*size;
- off = (uintptr*)p - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- bits = *bitp>>shift;
- if((bits & (bitAllocated|bitMarked)) == bitAllocated) {
- // Find the exact byte for which the special was setup
- // (as opposed to object beginning).
- p = (byte*)(s->start << PageShift) + special->offset;
- // about to free object: splice out special record
- y = special;
- special = special->next;
- *specialp = special;
- if(!runtime·freespecial(y, p, size, false)) {
- // stop freeing of object if it has a finalizer
- *bitp |= bitMarked << shift;
- }
- } else {
- // object is still live: keep special record
- specialp = &special->next;
- special = *specialp;
- }
- }
-
- type_data = (byte*)s->types.data;
- type_data_inc = sizeof(uintptr);
- compression = s->types.compression;
- switch(compression) {
- case MTypes_Bytes:
- type_data += 8*sizeof(uintptr);
- type_data_inc = 1;
- break;
- }
-
- // Sweep through n objects of given size starting at p.
- // This thread owns the span now, so it can manipulate
- // the block bitmap without atomic operations.
- p = (byte*)(s->start << PageShift);
- for(; n > 0; n--, p += size, type_data+=type_data_inc) {
- off = (uintptr*)p - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- bits = *bitp>>shift;
-
- if((bits & bitAllocated) == 0)
- continue;
-
- if((bits & bitMarked) != 0) {
- *bitp &= ~(bitMarked<<shift);
- continue;
- }
-
- if(runtime·debug.allocfreetrace)
- runtime·tracefree(p, size);
-
- // Clear mark and scan bits.
- *bitp &= ~((bitScan|bitMarked)<<shift);
-
- if(cl == 0) {
- // Free large span.
- runtime·unmarkspan(p, 1<<PageShift);
- s->needzero = 1;
- // important to set sweepgen before returning it to heap
- runtime·atomicstore(&s->sweepgen, sweepgen);
- sweepgenset = true;
- // See note about SysFault vs SysFree in malloc.goc.
- if(runtime·debug.efence)
- runtime·SysFault(p, size);
- else
- runtime·MHeap_Free(&runtime·mheap, s, 1);
- c->local_nlargefree++;
- c->local_largefree += size;
- runtime·xadd64(&mstats.next_gc, -(uint64)(size * (gcpercent + 100)/100));
- res = true;
- } else {
- // Free small object.
- switch(compression) {
- case MTypes_Words:
- *(uintptr*)type_data = 0;
- break;
- case MTypes_Bytes:
- *(byte*)type_data = 0;
- break;
- }
- if(size > 2*sizeof(uintptr))
- ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll; // mark as "needs to be zeroed"
- else if(size > sizeof(uintptr))
- ((uintptr*)p)[1] = 0;
-
- end->next = (MLink*)p;
- end = (MLink*)p;
- nfree++;
- }
- }
-
- // We need to set s->sweepgen = h->sweepgen only when all blocks are swept,
- // because of the potential for a concurrent free/SetFinalizer.
- // But we need to set it before we make the span available for allocation
- // (return it to heap or mcentral), because allocation code assumes that a
- // span is already swept if available for allocation.
-
- if(!sweepgenset && nfree == 0) {
- // The span must be in our exclusive ownership until we update sweepgen,
- // check for potential races.
- if(s->state != MSpanInUse || s->sweepgen != sweepgen-1) {
- runtime·printf("MSpan_Sweep: state=%d sweepgen=%d mheap.sweepgen=%d\n",
- s->state, s->sweepgen, sweepgen);
- runtime·throw("MSpan_Sweep: bad span state after sweep");
- }
- runtime·atomicstore(&s->sweepgen, sweepgen);
- }
- if(nfree > 0) {
- c->local_nsmallfree[cl] += nfree;
- c->local_cachealloc -= nfree * size;
- runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (gcpercent + 100)/100));
- res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end);
- //MCentral_FreeSpan updates sweepgen
- }
- return res;
-}
-
-// State of background sweep.
-// Pretected by gclock.
-static struct
-{
- G* g;
- bool parked;
-
- MSpan** spans;
- uint32 nspan;
- uint32 spanidx;
-} sweep;
-
-// background sweeping goroutine
-static void
-bgsweep(void)
-{
- g->issystem = 1;
- for(;;) {
- while(runtime·sweepone() != -1) {
- gcstats.nbgsweep++;
- runtime·gosched();
- }
- runtime·lock(&gclock);
- if(!runtime·mheap.sweepdone) {
- // It's possible if GC has happened between sweepone has
- // returned -1 and gclock lock.
- runtime·unlock(&gclock);
- continue;
- }
- sweep.parked = true;
- g->isbackground = true;
- runtime·parkunlock(&gclock, "GC sweep wait");
- g->isbackground = false;
- }
-}
-
-// sweeps one span
-// returns number of pages returned to heap, or -1 if there is nothing to sweep
-uintptr
-runtime·sweepone(void)
-{
- MSpan *s;
- uint32 idx, sg;
- uintptr npages;
-
- // increment locks to ensure that the goroutine is not preempted
- // in the middle of sweep thus leaving the span in an inconsistent state for next GC
- m->locks++;
- sg = runtime·mheap.sweepgen;
- for(;;) {
- idx = runtime·xadd(&sweep.spanidx, 1) - 1;
- if(idx >= sweep.nspan) {
- runtime·mheap.sweepdone = true;
- m->locks--;
- return -1;
- }
- s = sweep.spans[idx];
- if(s->state != MSpanInUse) {
- s->sweepgen = sg;
- continue;
- }
- if(s->sweepgen != sg-2 || !runtime·cas(&s->sweepgen, sg-2, sg-1))
- continue;
- if(s->incache)
- runtime·throw("sweep of incache span");
- npages = s->npages;
- if(!runtime·MSpan_Sweep(s))
- npages = 0;
- m->locks--;
- return npages;
- }
-}
-
-static void
-dumpspan(uint32 idx)
-{
- int32 sizeclass, n, npages, i, column;
- uintptr size;
- byte *p;
- byte *arena_start;
- MSpan *s;
- bool allocated;
-
- s = runtime·mheap.allspans[idx];
- if(s->state != MSpanInUse)
- return;
- arena_start = runtime·mheap.arena_start;
- p = (byte*)(s->start << PageShift);
- sizeclass = s->sizeclass;
- size = s->elemsize;
- if(sizeclass == 0) {
- n = 1;
- } else {
- npages = runtime·class_to_allocnpages[sizeclass];
- n = (npages << PageShift) / size;
- }
-
- runtime·printf("%p .. %p:\n", p, p+n*size);
- column = 0;
- for(; n>0; n--, p+=size) {
- uintptr off, *bitp, shift, bits;
-
- off = (uintptr*)p - (uintptr*)arena_start;
- bitp = (uintptr*)arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- bits = *bitp>>shift;
-
- allocated = ((bits & bitAllocated) != 0);
-
- for(i=0; i<size; i+=sizeof(void*)) {
- if(column == 0) {
- runtime·printf("\t");
- }
- if(i == 0) {
- runtime·printf(allocated ? "(" : "[");
- runtime·printf("%p: ", p+i);
- } else {
- runtime·printf(" ");
- }
-
- runtime·printf("%p", *(void**)(p+i));
-
- if(i+sizeof(void*) >= size) {
- runtime·printf(allocated ? ") " : "] ");
- }
-
- column++;
- if(column == 8) {
- runtime·printf("\n");
- column = 0;
- }
- }
- }
- runtime·printf("\n");
-}
-
-// A debugging function to dump the contents of memory
-void
-runtime·memorydump(void)
-{
- uint32 spanidx;
-
- for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
- dumpspan(spanidx);
- }
-}
-
-void
-runtime·gchelper(void)
-{
- uint32 nproc;
-
- m->traceback = 2;
- gchelperstart();
-
- // parallel mark for over gc roots
- runtime·parfordo(work.markfor);
-
- // help other threads scan secondary blocks
- scanblock(nil, true);
-
- bufferList[m->helpgc].busy = 0;
- nproc = work.nproc; // work.nproc can change right after we increment work.ndone
- if(runtime·xadd(&work.ndone, +1) == nproc-1)
- runtime·notewakeup(&work.alldone);
- m->traceback = 0;
-}
-
-static void
-cachestats(void)
-{
- MCache *c;
- P *p, **pp;
-
- for(pp=runtime·allp; p=*pp; pp++) {
- c = p->mcache;
- if(c==nil)
- continue;
- runtime·purgecachedstats(c);
- }
-}
-
-static void
-flushallmcaches(void)
-{
- P *p, **pp;
- MCache *c;
-
- // Flush MCache's to MCentral.
- for(pp=runtime·allp; p=*pp; pp++) {
- c = p->mcache;
- if(c==nil)
- continue;
- runtime·MCache_ReleaseAll(c);
- }
-}
-
-void
-runtime·updatememstats(GCStats *stats)
-{
- M *mp;
- MSpan *s;
- int32 i;
- uint64 stacks_inuse, smallfree;
- uint64 *src, *dst;
-
- if(stats)
- runtime·memclr((byte*)stats, sizeof(*stats));
- stacks_inuse = 0;
- for(mp=runtime·allm; mp; mp=mp->alllink) {
- stacks_inuse += mp->stackinuse*FixedStack;
- if(stats) {
- src = (uint64*)&mp->gcstats;
- dst = (uint64*)stats;
- for(i=0; i<sizeof(*stats)/sizeof(uint64); i++)
- dst[i] += src[i];
- runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
- }
- }
- mstats.stacks_inuse = stacks_inuse;
- mstats.mcache_inuse = runtime·mheap.cachealloc.inuse;
- mstats.mspan_inuse = runtime·mheap.spanalloc.inuse;
- mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys +
- mstats.mcache_sys + mstats.buckhash_sys + mstats.gc_sys + mstats.other_sys;
-
- // Calculate memory allocator stats.
- // During program execution we only count number of frees and amount of freed memory.
- // Current number of alive object in the heap and amount of alive heap memory
- // are calculated by scanning all spans.
- // Total number of mallocs is calculated as number of frees plus number of alive objects.
- // Similarly, total amount of allocated memory is calculated as amount of freed memory
- // plus amount of alive heap memory.
- mstats.alloc = 0;
- mstats.total_alloc = 0;
- mstats.nmalloc = 0;
- mstats.nfree = 0;
- for(i = 0; i < nelem(mstats.by_size); i++) {
- mstats.by_size[i].nmalloc = 0;
- mstats.by_size[i].nfree = 0;
- }
-
- // Flush MCache's to MCentral.
- flushallmcaches();
-
- // Aggregate local stats.
- cachestats();
-
- // Scan all spans and count number of alive objects.
- for(i = 0; i < runtime·mheap.nspan; i++) {
- s = runtime·mheap.allspans[i];
- if(s->state != MSpanInUse)
- continue;
- if(s->sizeclass == 0) {
- mstats.nmalloc++;
- mstats.alloc += s->elemsize;
- } else {
- mstats.nmalloc += s->ref;
- mstats.by_size[s->sizeclass].nmalloc += s->ref;
- mstats.alloc += s->ref*s->elemsize;
- }
- }
-
- // Aggregate by size class.
- smallfree = 0;
- mstats.nfree = runtime·mheap.nlargefree;
- for(i = 0; i < nelem(mstats.by_size); i++) {
- mstats.nfree += runtime·mheap.nsmallfree[i];
- mstats.by_size[i].nfree = runtime·mheap.nsmallfree[i];
- mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i];
- smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i];
- }
- mstats.nmalloc += mstats.nfree;
-
- // Calculate derived stats.
- mstats.total_alloc = mstats.alloc + runtime·mheap.largefree + smallfree;
- mstats.heap_alloc = mstats.alloc;
- mstats.heap_objects = mstats.nmalloc - mstats.nfree;
-}
-
-// Structure of arguments passed to function gc().
-// This allows the arguments to be passed via runtime·mcall.
-struct gc_args
-{
- int64 start_time; // start time of GC in ns (just before stoptheworld)
- bool eagersweep;
-};
-
-static void gc(struct gc_args *args);
-static void mgc(G *gp);
-
-static int32
-readgogc(void)
-{
- byte *p;
-
- p = runtime·getenv("GOGC");
- if(p == nil || p[0] == '\0')
- return 100;
- if(runtime·strcmp(p, (byte*)"off") == 0)
- return -1;
- return runtime·atoi(p);
-}
-
-// force = 1 - do GC regardless of current heap usage
-// force = 2 - go GC and eager sweep
-void
-runtime·gc(int32 force)
-{
- struct gc_args a;
- int32 i;
-
- // The atomic operations are not atomic if the uint64s
- // are not aligned on uint64 boundaries. This has been
- // a problem in the past.
- if((((uintptr)&work.empty) & 7) != 0)
- runtime·throw("runtime: gc work buffer is misaligned");
- if((((uintptr)&work.full) & 7) != 0)
- runtime·throw("runtime: gc work buffer is misaligned");
-
- // The gc is turned off (via enablegc) until
- // the bootstrap has completed.
- // Also, malloc gets called in the guts
- // of a number of libraries that might be
- // holding locks. To avoid priority inversion
- // problems, don't bother trying to run gc
- // while holding a lock. The next mallocgc
- // without a lock will do the gc instead.
- if(!mstats.enablegc || g == m->g0 || m->locks > 0 || runtime·panicking)
- return;
-
- if(gcpercent == GcpercentUnknown) { // first time through
- runtime·lock(&runtime·mheap);
- if(gcpercent == GcpercentUnknown)
- gcpercent = readgogc();
- runtime·unlock(&runtime·mheap);
- }
- if(gcpercent < 0)
- return;
-
- runtime·semacquire(&runtime·worldsema, false);
- if(force==0 && mstats.heap_alloc < mstats.next_gc) {
- // typically threads which lost the race to grab
- // worldsema exit here when gc is done.
- runtime·semrelease(&runtime·worldsema);
- return;
- }
-
- // Ok, we're doing it! Stop everybody else
- a.start_time = runtime·nanotime();
- a.eagersweep = force >= 2;
- m->gcing = 1;
- runtime·stoptheworld();
-
- clearpools();
-
- // Run gc on the g0 stack. We do this so that the g stack
- // we're currently running on will no longer change. Cuts
- // the root set down a bit (g0 stacks are not scanned, and
- // we don't need to scan gc's internal state). Also an
- // enabler for copyable stacks.
- for(i = 0; i < (runtime·debug.gctrace > 1 ? 2 : 1); i++) {
- if(i > 0)
- a.start_time = runtime·nanotime();
- // switch to g0, call gc(&a), then switch back
- g->param = &a;
- g->status = Gwaiting;
- g->waitreason = "garbage collection";
- runtime·mcall(mgc);
- }
-
- // all done
- m->gcing = 0;
- m->locks++;
- runtime·semrelease(&runtime·worldsema);
- runtime·starttheworld();
- m->locks--;
-
- // now that gc is done, kick off finalizer thread if needed
- if(!ConcurrentSweep) {
- // give the queued finalizers, if any, a chance to run
- runtime·gosched();
- }
-}
-
-static void
-mgc(G *gp)
-{
- gc(gp->param);
- gp->param = nil;
- gp->status = Grunning;
- runtime·gogo(&gp->sched);
-}
-
-static void
-gc(struct gc_args *args)
-{
- int64 t0, t1, t2, t3, t4;
- uint64 heap0, heap1, obj, ninstr;
- GCStats stats;
- uint32 i;
- Eface eface;
-
- if(runtime·debug.allocfreetrace)
- runtime·tracegc();
-
- m->traceback = 2;
- t0 = args->start_time;
- work.tstart = args->start_time;
-
- if(CollectStats)
- runtime·memclr((byte*)&gcstats, sizeof(gcstats));
-
- m->locks++; // disable gc during mallocs in parforalloc
- if(work.markfor == nil)
- work.markfor = runtime·parforalloc(MaxGcproc);
- m->locks--;
-
- if(itabtype == nil) {
- // get C pointer to the Go type "itab"
- runtime·gc_itab_ptr(&eface);
- itabtype = ((PtrType*)eface.type)->elem;
- }
-
- t1 = 0;
- if(runtime·debug.gctrace)
- t1 = runtime·nanotime();
-
- // Sweep what is not sweeped by bgsweep.
- while(runtime·sweepone() != -1)
- gcstats.npausesweep++;
-
- work.nwait = 0;
- work.ndone = 0;
- work.nproc = runtime·gcprocs();
- runtime·parforsetup(work.markfor, work.nproc, RootCount + runtime·allglen, nil, false, markroot);
- if(work.nproc > 1) {
- runtime·noteclear(&work.alldone);
- runtime·helpgc(work.nproc);
- }
-
- t2 = 0;
- if(runtime·debug.gctrace)
- t2 = runtime·nanotime();
-
- gchelperstart();
- runtime·parfordo(work.markfor);
- scanblock(nil, true);
-
- t3 = 0;
- if(runtime·debug.gctrace)
- t3 = runtime·nanotime();
-
- bufferList[m->helpgc].busy = 0;
- if(work.nproc > 1)
- runtime·notesleep(&work.alldone);
-
- cachestats();
- // next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
- // estimate what was live heap size after previous GC (for tracing only)
- heap0 = mstats.next_gc*100/(gcpercent+100);
- // conservatively set next_gc to high value assuming that everything is live
- // concurrent/lazy sweep will reduce this number while discovering new garbage
- mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
-
- t4 = runtime·nanotime();
- mstats.last_gc = runtime·unixnanotime(); // must be Unix time to make sense to user
- mstats.pause_ns[mstats.numgc%nelem(mstats.pause_ns)] = t4 - t0;
- mstats.pause_total_ns += t4 - t0;
- mstats.numgc++;
- if(mstats.debuggc)
- runtime·printf("pause %D\n", t4-t0);
-
- if(runtime·debug.gctrace) {
- heap1 = mstats.heap_alloc;
- runtime·updatememstats(&stats);
- if(heap1 != mstats.heap_alloc) {
- runtime·printf("runtime: mstats skew: heap=%D/%D\n", heap1, mstats.heap_alloc);
- runtime·throw("mstats skew");
- }
- obj = mstats.nmalloc - mstats.nfree;
-
- stats.nprocyield += work.markfor->nprocyield;
- stats.nosyield += work.markfor->nosyield;
- stats.nsleep += work.markfor->nsleep;
-
- runtime·printf("gc%d(%d): %D+%D+%D+%D us, %D -> %D MB, %D (%D-%D) objects,"
- " %d/%d/%d sweeps,"
- " %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
- mstats.numgc, work.nproc, (t1-t0)/1000, (t2-t1)/1000, (t3-t2)/1000, (t4-t3)/1000,
- heap0>>20, heap1>>20, obj,
- mstats.nmalloc, mstats.nfree,
- sweep.nspan, gcstats.nbgsweep, gcstats.npausesweep,
- stats.nhandoff, stats.nhandoffcnt,
- work.markfor->nsteal, work.markfor->nstealcnt,
- stats.nprocyield, stats.nosyield, stats.nsleep);
- gcstats.nbgsweep = gcstats.npausesweep = 0;
- if(CollectStats) {
- runtime·printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n",
- gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.notype, gcstats.obj.typelookup);
- if(gcstats.ptr.cnt != 0)
- runtime·printf("avg ptrbufsize: %D (%D/%D)\n",
- gcstats.ptr.sum/gcstats.ptr.cnt, gcstats.ptr.sum, gcstats.ptr.cnt);
- if(gcstats.obj.cnt != 0)
- runtime·printf("avg nobj: %D (%D/%D)\n",
- gcstats.obj.sum/gcstats.obj.cnt, gcstats.obj.sum, gcstats.obj.cnt);
- runtime·printf("rescans: %D, %D bytes\n", gcstats.rescan, gcstats.rescanbytes);
-
- runtime·printf("instruction counts:\n");
- ninstr = 0;
- for(i=0; i<nelem(gcstats.instr); i++) {
- runtime·printf("\t%d:\t%D\n", i, gcstats.instr[i]);
- ninstr += gcstats.instr[i];
- }
- runtime·printf("\ttotal:\t%D\n", ninstr);
-
- runtime·printf("putempty: %D, getfull: %D\n", gcstats.putempty, gcstats.getfull);
-
- runtime·printf("markonly base lookup: bit %D word %D span %D\n", gcstats.markonly.foundbit, gcstats.markonly.foundword, gcstats.markonly.foundspan);
- runtime·printf("flushptrbuf base lookup: bit %D word %D span %D\n", gcstats.flushptrbuf.foundbit, gcstats.flushptrbuf.foundword, gcstats.flushptrbuf.foundspan);
- }
- }
-
- // We cache current runtime·mheap.allspans array in sweep.spans,
- // because the former can be resized and freed.
- // Otherwise we would need to take heap lock every time
- // we want to convert span index to span pointer.
-
- // Free the old cached array if necessary.
- if(sweep.spans && sweep.spans != runtime·mheap.allspans)
- runtime·SysFree(sweep.spans, sweep.nspan*sizeof(sweep.spans[0]), &mstats.other_sys);
- // Cache the current array.
- runtime·mheap.sweepspans = runtime·mheap.allspans;
- runtime·mheap.sweepgen += 2;
- runtime·mheap.sweepdone = false;
- sweep.spans = runtime·mheap.allspans;
- sweep.nspan = runtime·mheap.nspan;
- sweep.spanidx = 0;
-
- // Temporary disable concurrent sweep, because we see failures on builders.
- if(ConcurrentSweep && !args->eagersweep) {
- runtime·lock(&gclock);
- if(sweep.g == nil)
- sweep.g = runtime·newproc1(&bgsweepv, nil, 0, 0, runtime·gc);
- else if(sweep.parked) {
- sweep.parked = false;
- runtime·ready(sweep.g);
- }
- runtime·unlock(&gclock);
- } else {
- // Sweep all spans eagerly.
- while(runtime·sweepone() != -1)
- gcstats.npausesweep++;
- }
-
- // Shrink a stack if not much of it is being used.
- // TODO: do in a parfor
- for(i = 0; i < runtime·allglen; i++)
- runtime·shrinkstack(runtime·allg[i]);
-
- runtime·MProf_GC();
- m->traceback = 0;
-}
-
-extern uintptr runtime·sizeof_C_MStats;
-
-void
-runtime·ReadMemStats(MStats *stats)
-{
- // Have to acquire worldsema to stop the world,
- // because stoptheworld can only be used by
- // one goroutine at a time, and there might be
- // a pending garbage collection already calling it.
- runtime·semacquire(&runtime·worldsema, false);
- m->gcing = 1;
- runtime·stoptheworld();
- runtime·updatememstats(nil);
- // Size of the trailing by_size array differs between Go and C,
- // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
- runtime·memcopy(runtime·sizeof_C_MStats, stats, &mstats);
- m->gcing = 0;
- m->locks++;
- runtime·semrelease(&runtime·worldsema);
- runtime·starttheworld();
- m->locks--;
-}
-
-void
-runtime∕debug·readGCStats(Slice *pauses)
-{
- uint64 *p;
- uint32 i, n;
-
- // Calling code in runtime/debug should make the slice large enough.
- if(pauses->cap < nelem(mstats.pause_ns)+3)
- runtime·throw("runtime: short slice passed to readGCStats");
-
- // Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
- p = (uint64*)pauses->array;
- runtime·lock(&runtime·mheap);
- n = mstats.numgc;
- if(n > nelem(mstats.pause_ns))
- n = nelem(mstats.pause_ns);
-
- // The pause buffer is circular. The most recent pause is at
- // pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
- // from there to go back farther in time. We deliver the times
- // most recent first (in p[0]).
- for(i=0; i<n; i++)
- p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)];
-
- p[n] = mstats.last_gc;
- p[n+1] = mstats.numgc;
- p[n+2] = mstats.pause_total_ns;
- runtime·unlock(&runtime·mheap);
- pauses->len = n+3;
-}
-
-int32
-runtime·setgcpercent(int32 in) {
- int32 out;
-
- runtime·lock(&runtime·mheap);
- if(gcpercent == GcpercentUnknown)
- gcpercent = readgogc();
- out = gcpercent;
- if(in < 0)
- in = -1;
- gcpercent = in;
- runtime·unlock(&runtime·mheap);
- return out;
-}
-
-static void
-gchelperstart(void)
-{
- if(m->helpgc < 0 || m->helpgc >= MaxGcproc)
- runtime·throw("gchelperstart: bad m->helpgc");
- if(runtime·xchg(&bufferList[m->helpgc].busy, 1))
- runtime·throw("gchelperstart: already busy");
- if(g != m->g0)
- runtime·throw("gchelper not running on g0 stack");
-}
-
-static void
-runfinq(void)
-{
- Finalizer *f;
- FinBlock *fb, *next;
- byte *frame;
- uint32 framesz, framecap, i;
- Eface *ef, ef1;
-
- // This function blocks for long periods of time, and because it is written in C
- // we have no liveness information. Zero everything so that uninitialized pointers
- // do not cause memory leaks.
- f = nil;
- fb = nil;
- next = nil;
- frame = nil;
- framecap = 0;
- framesz = 0;
- i = 0;
- ef = nil;
- ef1.type = nil;
- ef1.data = nil;
-
- // force flush to memory
- USED(&f);
- USED(&fb);
- USED(&next);
- USED(&framesz);
- USED(&i);
- USED(&ef);
- USED(&ef1);
-
- for(;;) {
- runtime·lock(&finlock);
- fb = finq;
- finq = nil;
- if(fb == nil) {
- runtime·fingwait = true;
- g->isbackground = true;
- runtime·parkunlock(&finlock, "finalizer wait");
- g->isbackground = false;
- continue;
- }
- runtime·unlock(&finlock);
- if(raceenabled)
- runtime·racefingo();
- for(; fb; fb=next) {
- next = fb->next;
- for(i=0; i<fb->cnt; i++) {
- f = &fb->fin[i];
- framesz = sizeof(Eface) + f->nret;
- if(framecap < framesz) {
- runtime·free(frame);
- // The frame does not contain pointers interesting for GC,
- // all not yet finalized objects are stored in finq.
- // If we do not mark it as FlagNoScan,
- // the last finalized object is not collected.
- frame = runtime·mallocgc(framesz, 0, FlagNoScan|FlagNoInvokeGC);
- framecap = framesz;
- }
- if(f->fint == nil)
- runtime·throw("missing type in runfinq");
- if(f->fint->kind == KindPtr) {
- // direct use of pointer
- *(void**)frame = f->arg;
- } else if(((InterfaceType*)f->fint)->mhdr.len == 0) {
- // convert to empty interface
- ef = (Eface*)frame;
- ef->type = f->ot;
- ef->data = f->arg;
- } else {
- // convert to interface with methods, via empty interface.
- ef1.type = f->ot;
- ef1.data = f->arg;
- if(!runtime·ifaceE2I2((InterfaceType*)f->fint, ef1, (Iface*)frame))
- runtime·throw("invalid type conversion in runfinq");
- }
- reflect·call(f->fn, frame, framesz, framesz);
- f->fn = nil;
- f->arg = nil;
- f->ot = nil;
- }
- fb->cnt = 0;
- runtime·lock(&finlock);
- fb->next = finc;
- finc = fb;
- runtime·unlock(&finlock);
- }
-
- // Zero everything that's dead, to avoid memory leaks.
- // See comment at top of function.
- f = nil;
- fb = nil;
- next = nil;
- i = 0;
- ef = nil;
- ef1.type = nil;
- ef1.data = nil;
- runtime·gc(1); // trigger another gc to clean up the finalized objects, if possible
- }
-}
-
-void
-runtime·createfing(void)
-{
- if(fing != nil)
- return;
- // Here we use gclock instead of finlock,
- // because newproc1 can allocate, which can cause on-demand span sweep,
- // which can queue finalizers, which would deadlock.
- runtime·lock(&gclock);
- if(fing == nil)
- fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
- runtime·unlock(&gclock);
-}
-
-G*
-runtime·wakefing(void)
-{
- G *res;
-
- res = nil;
- runtime·lock(&finlock);
- if(runtime·fingwait && runtime·fingwake) {
- runtime·fingwait = false;
- runtime·fingwake = false;
- res = fing;
- }
- runtime·unlock(&finlock);
- return res;
-}
-
-void
-runtime·marknogc(void *v)
-{
- uintptr *b, off, shift;
-
- off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
- b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- *b = (*b & ~(bitAllocated<<shift)) | bitBlockBoundary<<shift;
-}
-
-void
-runtime·markscan(void *v)
-{
- uintptr *b, off, shift;
-
- off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
- b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- *b |= bitScan<<shift;
-}
-
-// mark the block at v as freed.
-void
-runtime·markfreed(void *v)
-{
- uintptr *b, off, shift;
-
- if(0)
- runtime·printf("markfreed %p\n", v);
-
- if((byte*)v > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
- runtime·throw("markfreed: bad pointer");
-
- off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
- b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- *b = (*b & ~(bitMask<<shift)) | (bitAllocated<<shift);
-}
-
-// check that the block at v of size n is marked freed.
-void
-runtime·checkfreed(void *v, uintptr n)
-{
- uintptr *b, bits, off, shift;
-
- if(!runtime·checking)
- return;
-
- if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
- return; // not allocated, so okay
-
- off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
- b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
-
- bits = *b>>shift;
- if((bits & bitAllocated) != 0) {
- runtime·printf("checkfreed %p+%p: off=%p have=%p\n",
- v, n, off, bits & bitMask);
- runtime·throw("checkfreed: not freed");
- }
-}
-
-// mark the span of memory at v as having n blocks of the given size.
-// if leftover is true, there is left over space at the end of the span.
-void
-runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
-{
- uintptr *b, *b0, off, shift, i, x;
- byte *p;
-
- if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
- runtime·throw("markspan: bad pointer");
-
- if(runtime·checking) {
- // bits should be all zero at the start
- off = (byte*)v + size - runtime·mheap.arena_start;
- b = (uintptr*)(runtime·mheap.arena_start - off/wordsPerBitmapWord);
- for(i = 0; i < size/PtrSize/wordsPerBitmapWord; i++) {
- if(b[i] != 0)
- runtime·throw("markspan: span bits not zero");
- }
- }
-
- p = v;
- if(leftover) // mark a boundary just past end of last block too
- n++;
-
- b0 = nil;
- x = 0;
- for(; n-- > 0; p += size) {
- // Okay to use non-atomic ops here, because we control
- // the entire span, and each bitmap word has bits for only
- // one span, so no other goroutines are changing these
- // bitmap words.
- off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // word offset
- b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- shift = off % wordsPerBitmapWord;
- if(b0 != b) {
- if(b0 != nil)
- *b0 = x;
- b0 = b;
- x = 0;
- }
- x |= bitAllocated<<shift;
- }
- *b0 = x;
-}
-
-// unmark the span of memory at v of length n bytes.
-void
-runtime·unmarkspan(void *v, uintptr n)
-{
- uintptr *p, *b, off;
-
- if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
- runtime·throw("markspan: bad pointer");
-
- p = v;
- off = p - (uintptr*)runtime·mheap.arena_start; // word offset
- if(off % wordsPerBitmapWord != 0)
- runtime·throw("markspan: unaligned pointer");
- b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
- n /= PtrSize;
- if(n%wordsPerBitmapWord != 0)
- runtime·throw("unmarkspan: unaligned length");
- // Okay to use non-atomic ops here, because we control
- // the entire span, and each bitmap word has bits for only
- // one span, so no other goroutines are changing these
- // bitmap words.
- n /= wordsPerBitmapWord;
- while(n-- > 0)
- *b-- = 0;
-}
-
-void
-runtime·MHeap_MapBits(MHeap *h)
-{
- // Caller has added extra mappings to the arena.
- // Add extra mappings of bitmap words as needed.
- // We allocate extra bitmap pieces in chunks of bitmapChunk.
- enum {
- bitmapChunk = 8192
- };
- uintptr n;
-
- n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
- n = ROUND(n, bitmapChunk);
- n = ROUND(n, PhysPageSize);
- if(h->bitmap_mapped >= n)
- return;
-
- runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, h->arena_reserved, &mstats.gc_sys);
- h->bitmap_mapped = n;
-}
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-17 12:03:55 +0000