summaryrefslogtreecommitdiff
path: root/src/pkg/runtime/mgc0.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/runtime/mgc0.c')
-rw-r--r--src/pkg/runtime/mgc0.c804
1 files changed, 458 insertions, 346 deletions
diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index c83f1892c..4b2108ba7 100644
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -12,7 +12,8 @@
#include "race.h"
#include "type.h"
#include "typekind.h"
-#include "hashmap.h"
+#include "funcdata.h"
+#include "../../cmd/ld/textflag.h"
enum {
Debug = 0,
@@ -32,6 +33,13 @@ enum {
PRECISE = 1,
LOOP = 2,
PC_BITS = PRECISE | LOOP,
+
+ // Pointer map
+ BitsPerPointer = 2,
+ BitsNoPointer = 0,
+ BitsPointer = 1,
+ BitsIface = 2,
+ BitsEface = 3,
};
// Bits in per-word bitmap.
@@ -43,7 +51,7 @@ enum {
// The bits in the word are packed together by type first, then by
// heap location, so each 64-bit bitmap word consists of, from top to bottom,
// the 16 bitSpecial bits for the corresponding heap words, then the 16 bitMarked bits,
-// then the 16 bitNoPointers/bitBlockBoundary bits, then the 16 bitAllocated bits.
+// then the 16 bitNoScan/bitBlockBoundary bits, then the 16 bitAllocated bits.
// This layout makes it easier to iterate over the bits of a given type.
//
// The bitmap starts at mheap.arena_start and extends *backward* from
@@ -60,7 +68,7 @@ enum {
// /* then test bits & bitAllocated, bits & bitMarked, etc. */
//
#define bitAllocated ((uintptr)1<<(bitShift*0))
-#define bitNoPointers ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
+#define bitNoScan ((uintptr)1<<(bitShift*1)) /* when bitAllocated is set */
#define bitMarked ((uintptr)1<<(bitShift*2)) /* when bitAllocated is set */
#define bitSpecial ((uintptr)1<<(bitShift*3)) /* when bitAllocated is set - has finalizer or being profiled */
#define bitBlockBoundary ((uintptr)1<<(bitShift*1)) /* when bitAllocated is NOT set */
@@ -82,8 +90,6 @@ enum {
//
uint32 runtime·worldsema = 1;
-static int32 gctrace;
-
typedef struct Obj Obj;
struct Obj
{
@@ -110,6 +116,8 @@ struct Finalizer
FuncVal *fn;
void *arg;
uintptr nret;
+ Type *fint;
+ PtrType *ot;
};
typedef struct FinBlock FinBlock;
@@ -167,7 +175,6 @@ static struct {
enum {
GC_DEFAULT_PTR = GC_NUM_INSTR,
- GC_MAP_NEXT,
GC_CHAN,
GC_NUM_INSTR2
@@ -190,6 +197,16 @@ static struct {
uint64 instr[GC_NUM_INSTR2];
uint64 putempty;
uint64 getfull;
+ struct {
+ uint64 foundbit;
+ uint64 foundword;
+ uint64 foundspan;
+ } flushptrbuf;
+ struct {
+ uint64 foundbit;
+ uint64 foundword;
+ uint64 foundspan;
+ } markonly;
} gcstats;
// markonly marks an object. It returns true if the object
@@ -199,12 +216,12 @@ static bool
markonly(void *obj)
{
byte *p;
- uintptr *bitp, bits, shift, x, xbits, off;
+ uintptr *bitp, bits, shift, x, xbits, off, j;
MSpan *s;
PageID k;
// Words outside the arena cannot be pointers.
- if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
+ if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
return false;
// obj may be a pointer to a live object.
@@ -214,42 +231,57 @@ markonly(void *obj)
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
// Find bits for this word.
- off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start;
- bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
+ bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
// Pointing at the beginning of a block?
- if((bits & (bitAllocated|bitBlockBoundary)) != 0)
+ if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
+ if(CollectStats)
+ runtime·xadd64(&gcstats.markonly.foundbit, 1);
goto found;
+ }
+
+ // Pointing just past the beginning?
+ // Scan backward a little to find a block boundary.
+ for(j=shift; j-->0; ) {
+ if(((xbits>>j) & (bitAllocated|bitBlockBoundary)) != 0) {
+ shift = j;
+ bits = xbits>>shift;
+ if(CollectStats)
+ runtime·xadd64(&gcstats.markonly.foundword, 1);
+ goto found;
+ }
+ }
// Otherwise consult span table to find beginning.
// (Manually inlined copy of MHeap_LookupMaybe.)
k = (uintptr)obj>>PageShift;
x = k;
if(sizeof(void*) == 8)
- x -= (uintptr)runtime·mheap->arena_start>>PageShift;
- s = runtime·mheap->map[x];
- if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
+ x -= (uintptr)runtime·mheap.arena_start>>PageShift;
+ s = runtime·mheap.spans[x];
+ if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
return false;
p = (byte*)((uintptr)s->start<<PageShift);
if(s->sizeclass == 0) {
obj = p;
} else {
- if((byte*)obj >= (byte*)s->limit)
- return false;
uintptr size = s->elemsize;
int32 i = ((byte*)obj - p)/size;
obj = p+i*size;
}
// Now that we know the object header, reload bits.
- off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start;
- bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
+ bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
+ if(CollectStats)
+ runtime·xadd64(&gcstats.markonly.foundspan, 1);
found:
// Now we have bits, bitp, and shift correct for
@@ -293,7 +325,7 @@ struct BufferList
uint32 busy;
byte pad[CacheLineSize];
};
-#pragma dataflag 16 // no pointers
+#pragma dataflag NOPTR
static BufferList bufferList[MaxGcproc];
static Type *itabtype;
@@ -328,7 +360,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
Workbuf *wbuf;
PtrTarget *ptrbuf_end;
- arena_start = runtime·mheap->arena_start;
+ arena_start = runtime·mheap.arena_start;
wp = *_wp;
wbuf = *_wbuf;
@@ -367,7 +399,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
// obj belongs to interval [mheap.arena_start, mheap.arena_used).
if(Debug > 1) {
- if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
+ if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
runtime·throw("object is outside of mheap");
}
@@ -388,8 +420,11 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
bits = xbits >> shift;
// Pointing at the beginning of a block?
- if((bits & (bitAllocated|bitBlockBoundary)) != 0)
+ if((bits & (bitAllocated|bitBlockBoundary)) != 0) {
+ if(CollectStats)
+ runtime·xadd64(&gcstats.flushptrbuf.foundbit, 1);
goto found;
+ }
ti = 0;
@@ -400,6 +435,8 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
obj = (byte*)obj - (shift-j)*PtrSize;
shift = j;
bits = xbits>>shift;
+ if(CollectStats)
+ runtime·xadd64(&gcstats.flushptrbuf.foundword, 1);
goto found;
}
}
@@ -410,15 +447,13 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
x = k;
if(sizeof(void*) == 8)
x -= (uintptr)arena_start>>PageShift;
- s = runtime·mheap->map[x];
- if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
+ s = runtime·mheap.spans[x];
+ if(s == nil || k < s->start || obj >= s->limit || s->state != MSpanInUse)
continue;
p = (byte*)((uintptr)s->start<<PageShift);
if(s->sizeclass == 0) {
obj = p;
} else {
- if((byte*)obj >= (byte*)s->limit)
- continue;
size = s->elemsize;
int32 i = ((byte*)obj - p)/size;
obj = p+i*size;
@@ -430,6 +465,8 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
+ if(CollectStats)
+ runtime·xadd64(&gcstats.flushptrbuf.foundspan, 1);
found:
// Now we have bits, bitp, and shift correct for
@@ -450,7 +487,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
}
// If object has no pointers, don't need to scan further.
- if((bits & bitNoPointers) != 0)
+ if((bits & bitNoScan) != 0)
continue;
// Ask span about size class.
@@ -458,7 +495,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
x = (uintptr)obj >> PageShift;
if(sizeof(void*) == 8)
x -= (uintptr)arena_start>>PageShift;
- s = runtime·mheap->map[x];
+ s = runtime·mheap.spans[x];
PREFETCH(obj);
@@ -541,9 +578,6 @@ flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_
// Program that scans the whole block and treats every block element as a potential pointer
static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
-// Hashmap iterator program
-static uintptr mapProg[2] = {0, GC_MAP_NEXT};
-
// Hchan program
static uintptr chanProg[2] = {0, GC_CHAN};
@@ -566,7 +600,7 @@ checkptr(void *obj, uintptr objti)
if(!Debug)
runtime·throw("checkptr is debug only");
- if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
+ if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
return;
type = runtime·gettype(obj);
t = (Type*)(type & ~(uintptr)(PtrSize-1));
@@ -574,8 +608,8 @@ checkptr(void *obj, uintptr objti)
return;
x = (uintptr)obj >> PageShift;
if(sizeof(void*) == 8)
- x -= (uintptr)(runtime·mheap->arena_start)>>PageShift;
- s = runtime·mheap->map[x];
+ x -= (uintptr)(runtime·mheap.arena_start)>>PageShift;
+ s = runtime·mheap.spans[x];
objstart = (byte*)((uintptr)s->start<<PageShift);
if(s->sizeclass != 0) {
i = ((byte*)obj - objstart)/s->elemsize;
@@ -583,8 +617,11 @@ checkptr(void *obj, uintptr objti)
}
tisize = *(uintptr*)objti;
// Sanity check for object size: it should fit into the memory block.
- if((byte*)obj + tisize > objstart + s->elemsize)
+ if((byte*)obj + tisize > objstart + s->elemsize) {
+ runtime·printf("object of type '%S' at %p/%p does not fit in block %p/%p\n",
+ *t->string, obj, tisize, objstart, s->elemsize);
runtime·throw("invalid gc type info");
+ }
if(obj != objstart)
return;
// If obj points to the beginning of the memory block,
@@ -600,7 +637,7 @@ checkptr(void *obj, uintptr objti)
for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) {
if(pc1[j] != pc2[j]) {
runtime·printf("invalid gc type info for '%s' at %p, type info %p, block info %p\n",
- t->string ? (int8*)t->string->str : (int8*)"?", j, pc1[j], pc2[j]);
+ t->string ? (int8*)t->string->str : (int8*)"?", j, pc1[j], pc2[j]);
runtime·throw("invalid gc type info");
}
}
@@ -623,7 +660,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
byte *b, *arena_start, *arena_used;
uintptr n, i, end_b, elemsize, size, ti, objti, count, type;
uintptr *pc, precise_type, nominal_size;
- uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ret, chancap;
+ uintptr *chan_ret, chancap;
void *obj;
Type *t;
Slice *sliceptr;
@@ -633,11 +670,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
Obj *objbuf, *objbuf_end, *objbufpos;
Eface *eface;
Iface *iface;
- Hmap *hmap;
- MapType *maptype;
- bool mapkey_kind, mapval_kind;
- struct hash_gciter map_iter;
- struct hash_gciter_data d;
Hchan *chan;
ChanType *chantype;
@@ -645,8 +677,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
runtime·throw("scanblock: size of Workbuf is suboptimal");
// Memory arena parameters.
- arena_start = runtime·mheap->arena_start;
- arena_used = runtime·mheap->arena_used;
+ arena_start = runtime·mheap.arena_start;
+ arena_used = runtime·mheap.arena_used;
stack_ptr = stack+nelem(stack)-1;
@@ -666,10 +698,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
objbufpos = objbuf;
// (Silence the compiler)
- map_ret = nil;
- mapkey_size = mapval_size = 0;
- mapkey_kind = mapval_kind = false;
- mapkey_ti = mapval_ti = 0;
chan = nil;
chantype = nil;
chan_ret = nil;
@@ -738,23 +766,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
stack_top.elemsize = pc[0];
stack_top.loop_or_ret = pc+1;
break;
- case TypeInfo_Map:
- hmap = (Hmap*)b;
- maptype = (MapType*)t;
- if(hash_gciter_init(hmap, &map_iter)) {
- mapkey_size = maptype->key->size;
- mapkey_kind = maptype->key->kind;
- mapkey_ti = (uintptr)maptype->key->gc | PRECISE;
- mapval_size = maptype->elem->size;
- mapval_kind = maptype->elem->kind;
- mapval_ti = (uintptr)maptype->elem->gc | PRECISE;
-
- map_ret = nil;
- pc = mapProg;
- } else {
- goto next_block;
- }
- break;
case TypeInfo_Chan:
chan = (Hchan*)b;
chantype = (ChanType*)t;
@@ -955,77 +966,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
pc = (uintptr*)((byte*)pc + *(int32*)(pc+2)); // target of the CALL instruction
continue;
- case GC_MAP_PTR:
- hmap = *(Hmap**)(stack_top.b + pc[1]);
- if(hmap == nil) {
- pc += 3;
- continue;
- }
- if(markonly(hmap)) {
- maptype = (MapType*)pc[2];
- if(hash_gciter_init(hmap, &map_iter)) {
- mapkey_size = maptype->key->size;
- mapkey_kind = maptype->key->kind;
- mapkey_ti = (uintptr)maptype->key->gc | PRECISE;
- mapval_size = maptype->elem->size;
- mapval_kind = maptype->elem->kind;
- mapval_ti = (uintptr)maptype->elem->gc | PRECISE;
-
- // Start mapProg.
- map_ret = pc+3;
- pc = mapProg+1;
- } else {
- pc += 3;
- }
- } else {
- pc += 3;
- }
- continue;
-
- case GC_MAP_NEXT:
- // Add all keys and values to buffers, mark all subtables.
- while(hash_gciter_next(&map_iter, &d)) {
- // buffers: reserve space for 2 objects.
- if(ptrbufpos+2 >= ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
- if(objbufpos+2 >= objbuf_end)
- flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
-
- if(d.st != nil)
- markonly(d.st);
-
- if(d.key_data != nil) {
- if(!(mapkey_kind & KindNoPointers) || d.indirectkey) {
- if(!d.indirectkey)
- *objbufpos++ = (Obj){d.key_data, mapkey_size, mapkey_ti};
- else {
- if(Debug) {
- obj = *(void**)d.key_data;
- if(!(arena_start <= obj && obj < arena_used))
- runtime·throw("scanblock: inconsistent hashmap");
- }
- *ptrbufpos++ = (PtrTarget){*(void**)d.key_data, mapkey_ti};
- }
- }
- if(!(mapval_kind & KindNoPointers) || d.indirectval) {
- if(!d.indirectval)
- *objbufpos++ = (Obj){d.val_data, mapval_size, mapval_ti};
- else {
- if(Debug) {
- obj = *(void**)d.val_data;
- if(!(arena_start <= obj && obj < arena_used))
- runtime·throw("scanblock: inconsistent hashmap");
- }
- *ptrbufpos++ = (PtrTarget){*(void**)d.val_data, mapval_ti};
- }
- }
- }
- }
- if(map_ret == nil)
- goto next_block;
- pc = map_ret;
- continue;
-
case GC_REGION:
obj = (void*)(stack_top.b + pc[1]);
size = pc[2];
@@ -1038,7 +978,6 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
continue;
case GC_CHAN_PTR:
- // Similar to GC_MAP_PTR
chan = *(Hchan**)(stack_top.b + pc[1]);
if(chan == nil) {
pc += 3;
@@ -1157,14 +1096,14 @@ debug_scanblock(byte *b, uintptr n)
obj = (byte*)vp[i];
// Words outside the arena cannot be pointers.
- if((byte*)obj < runtime·mheap->arena_start || (byte*)obj >= runtime·mheap->arena_used)
+ if((byte*)obj < runtime·mheap.arena_start || (byte*)obj >= runtime·mheap.arena_used)
continue;
// Round down to word boundary.
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
// Consult span table to find beginning.
- s = runtime·MHeap_LookupMaybe(runtime·mheap, obj);
+ s = runtime·MHeap_LookupMaybe(&runtime·mheap, obj);
if(s == nil)
continue;
@@ -1173,15 +1112,13 @@ debug_scanblock(byte *b, uintptr n)
if(s->sizeclass == 0) {
obj = p;
} else {
- if((byte*)obj >= (byte*)s->limit)
- continue;
int32 i = ((byte*)obj - p)/size;
obj = p+i*size;
}
// Now that we know the object header, reload bits.
- off = (uintptr*)obj - (uintptr*)runtime·mheap->arena_start;
- bitp = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)obj - (uintptr*)runtime·mheap.arena_start;
+ bitp = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
xbits = *bitp;
bits = xbits >> shift;
@@ -1196,7 +1133,7 @@ debug_scanblock(byte *b, uintptr n)
runtime·printf("found unmarked block %p in %p\n", obj, vp+i);
// If object has no pointers, don't need to scan further.
- if((bits & bitNoPointers) != 0)
+ if((bits & bitNoScan) != 0)
continue;
debug_scanblock(obj, size);
@@ -1286,7 +1223,7 @@ getempty(Workbuf *b)
runtime·lock(&work);
if(work.nchunk < sizeof *b) {
work.nchunk = 1<<20;
- work.chunk = runtime·SysAlloc(work.nchunk);
+ work.chunk = runtime·SysAlloc(work.nchunk, &mstats.gc_sys);
if(work.chunk == nil)
runtime·throw("runtime: cannot allocate memory");
}
@@ -1377,12 +1314,12 @@ addroot(Obj obj)
cap = PageSize/sizeof(Obj);
if(cap < 2*work.rootcap)
cap = 2*work.rootcap;
- new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj));
+ new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj), &mstats.gc_sys);
if(new == nil)
runtime·throw("runtime: cannot allocate memory");
if(work.roots != nil) {
runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj));
- runtime·SysFree(work.roots, work.rootcap*sizeof(Obj));
+ runtime·SysFree(work.roots, work.rootcap*sizeof(Obj), &mstats.gc_sys);
}
work.roots = new;
work.rootcap = cap;
@@ -1391,26 +1328,106 @@ addroot(Obj obj)
work.nroot++;
}
-// Scan a stack frame. The doframe parameter is a signal that the previously
-// scanned activation has an unknown argument size. When *doframe is true the
-// current activation must have its entire frame scanned. Otherwise, only the
-// locals need to be scanned.
+extern byte pclntab[]; // base for f->ptrsoff
+
+typedef struct BitVector BitVector;
+struct BitVector
+{
+ int32 n;
+ uint32 data[];
+};
+
+// Scans an interface data value when the interface type indicates
+// that it is a pointer.
static void
-addframeroots(Func *f, byte*, byte *sp, void *doframe)
+scaninterfacedata(uintptr bits, byte *scanp, bool afterprologue)
{
- uintptr outs;
-
- if(thechar == '5')
- sp += sizeof(uintptr);
- if(f->locals == 0 || *(bool*)doframe == true)
- addroot((Obj){sp, f->frame - sizeof(uintptr), 0});
- else if(f->locals > 0) {
- outs = f->frame - sizeof(uintptr) - f->locals;
- addroot((Obj){sp + outs, f->locals, 0});
+ Itab *tab;
+ Type *type;
+
+ if(runtime·precisestack && afterprologue) {
+ if(bits == BitsIface) {
+ tab = *(Itab**)scanp;
+ if(tab->type->size <= sizeof(void*) && (tab->type->kind & KindNoPointers))
+ return;
+ } else { // bits == BitsEface
+ type = *(Type**)scanp;
+ if(type->size <= sizeof(void*) && (type->kind & KindNoPointers))
+ return;
+ }
+ }
+ addroot((Obj){scanp+PtrSize, PtrSize, 0});
+}
+
+// Starting from scanp, scans words corresponding to set bits.
+static void
+scanbitvector(byte *scanp, BitVector *bv, bool afterprologue)
+{
+ uintptr word, bits;
+ uint32 *wordp;
+ int32 i, remptrs;
+
+ wordp = bv->data;
+ for(remptrs = bv->n; remptrs > 0; remptrs -= 32) {
+ word = *wordp++;
+ if(remptrs < 32)
+ i = remptrs;
+ else
+ i = 32;
+ i /= BitsPerPointer;
+ for(; i > 0; i--) {
+ bits = word & 3;
+ if(bits != BitsNoPointer && *(void**)scanp != nil)
+ if(bits == BitsPointer)
+ addroot((Obj){scanp, PtrSize, 0});
+ else
+ scaninterfacedata(bits, scanp, afterprologue);
+ word >>= BitsPerPointer;
+ scanp += PtrSize;
+ }
}
- if(f->args > 0)
- addroot((Obj){sp + f->frame, f->args, 0});
- *(bool*)doframe = (f->args == ArgsSizeUnknown);
+}
+
+// Scan a stack frame: local variables and function arguments/results.
+static void
+addframeroots(Stkframe *frame, void*)
+{
+ Func *f;
+ BitVector *args, *locals;
+ uintptr size;
+ bool afterprologue;
+
+ f = frame->fn;
+
+ // Scan local variables if stack frame has been allocated.
+ // Use pointer information if known.
+ afterprologue = (frame->varp > (byte*)frame->sp);
+ if(afterprologue) {
+ locals = runtime·funcdata(f, FUNCDATA_GCLocals);
+ if(locals == nil) {
+ // No locals information, scan everything.
+ size = frame->varp - (byte*)frame->sp;
+ addroot((Obj){frame->varp - size, size, 0});
+ } else if(locals->n < 0) {
+ // Locals size information, scan just the
+ // locals.
+ size = -locals->n;
+ addroot((Obj){frame->varp - size, size, 0});
+ } else if(locals->n > 0) {
+ // Locals bitmap information, scan just the
+ // pointers in locals.
+ size = (locals->n*PtrSize) / BitsPerPointer;
+ scanbitvector(frame->varp - size, locals, afterprologue);
+ }
+ }
+
+ // Scan arguments.
+ // Use pointer information if known.
+ args = runtime·funcdata(f, FUNCDATA_GCArgs);
+ if(args != nil && args->n > 0)
+ scanbitvector(frame->argp, args, false);
+ else
+ addroot((Obj){frame->argp, frame->arglen, 0});
}
static void
@@ -1419,62 +1436,54 @@ addstackroots(G *gp)
M *mp;
int32 n;
Stktop *stk;
- byte *sp, *guard, *pc;
- Func *f;
- bool doframe;
+ uintptr sp, guard, pc, lr;
+ void *base;
+ uintptr size;
stk = (Stktop*)gp->stackbase;
- guard = (byte*)gp->stackguard;
-
- if(gp == g) {
- // Scanning our own stack: start at &gp.
- sp = runtime·getcallersp(&gp);
- pc = runtime·getcallerpc(&gp);
- } else if((mp = gp->m) != nil && mp->helpgc) {
- // gchelper's stack is in active use and has no interesting pointers.
- return;
- } else if(gp->gcstack != (uintptr)nil) {
+ guard = gp->stackguard;
+
+ if(gp == g)
+ runtime·throw("can't scan our own stack");
+ if((mp = gp->m) != nil && mp->helpgc)
+ runtime·throw("can't scan gchelper stack");
+ if(gp->syscallstack != (uintptr)nil) {
// Scanning another goroutine that is about to enter or might
// have just exited a system call. It may be executing code such
// as schedlock and may have needed to start a new stack segment.
// Use the stack segment and stack pointer at the time of
// the system call instead, since that won't change underfoot.
- sp = (byte*)gp->gcsp;
- pc = gp->gcpc;
- stk = (Stktop*)gp->gcstack;
- guard = (byte*)gp->gcguard;
+ sp = gp->syscallsp;
+ pc = gp->syscallpc;
+ lr = 0;
+ stk = (Stktop*)gp->syscallstack;
+ guard = gp->syscallguard;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
- sp = (byte*)gp->sched.sp;
+ sp = gp->sched.sp;
pc = gp->sched.pc;
- if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnstart != nil) {
- // The goroutine has not started. However, its incoming
- // arguments are live at the top of the stack and must
- // be scanned. No other live values should be on the
- // stack.
- f = runtime·findfunc((uintptr)gp->fnstart->fn);
- if(f->args > 0) {
- if(thechar == '5')
- sp += sizeof(uintptr);
- addroot((Obj){sp, f->args, 0});
- }
- return;
- }
+ lr = gp->sched.lr;
+
+ // For function about to start, context argument is a root too.
+ if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
+ addroot((Obj){base, size, 0});
}
- if (ScanStackByFrames) {
- doframe = false;
- runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addframeroots, &doframe);
+ if(ScanStackByFrames) {
+ USED(stk);
+ USED(guard);
+ runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, addframeroots, nil, false);
} else {
+ USED(lr);
USED(pc);
n = 0;
while(stk) {
- if(sp < guard-StackGuard || (byte*)stk < sp) {
+ if(sp < guard-StackGuard || (uintptr)stk < sp) {
runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
runtime·throw("scanstack");
}
- addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
- sp = (byte*)stk->gobuf.sp;
+ addroot((Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
+ sp = stk->gobuf.sp;
guard = stk->stackguard;
stk = (Stktop*)stk->stackbase;
n++;
@@ -1512,8 +1521,8 @@ addroots(void)
addroot((Obj){bss, ebss - bss, (uintptr)gcbss});
// MSpan.types
- allspans = runtime·mheap->allspans;
- for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) {
+ allspans = runtime·mheap.allspans;
+ for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
s = allspans[spanidx];
if(s->state == MSpanInUse) {
// The garbage collector ignores type pointers stored in MSpan.types:
@@ -1541,10 +1550,7 @@ addroots(void)
case Gdead:
break;
case Grunning:
- if(gp != g)
- runtime·throw("mark - world not stopped");
- addstackroots(gp);
- break;
+ runtime·throw("mark - world not stopped");
case Grunnable:
case Gsyscall:
case Gwaiting:
@@ -1564,10 +1570,12 @@ handlespecial(byte *p, uintptr size)
{
FuncVal *fn;
uintptr nret;
+ PtrType *ot;
+ Type *fint;
FinBlock *block;
Finalizer *f;
- if(!runtime·getfinalizer(p, true, &fn, &nret)) {
+ if(!runtime·getfinalizer(p, true, &fn, &nret, &fint, &ot)) {
runtime·setblockspecial(p, false);
runtime·MProf_Free(p, size);
return false;
@@ -1576,9 +1584,7 @@ handlespecial(byte *p, uintptr size)
runtime·lock(&finlock);
if(finq == nil || finq->cnt == finq->cap) {
if(finc == nil) {
- finc = runtime·SysAlloc(PageSize);
- if(finc == nil)
- runtime·throw("runtime: cannot allocate memory");
+ finc = runtime·persistentalloc(PageSize, 0, &mstats.gc_sys);
finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1;
finc->alllink = allfin;
allfin = finc;
@@ -1592,6 +1598,8 @@ handlespecial(byte *p, uintptr size)
finq->cnt++;
f->fn = fn;
f->nret = nret;
+ f->fint = fint;
+ f->ot = ot;
f->arg = p;
runtime·unlock(&finlock);
return true;
@@ -1615,10 +1623,10 @@ sweepspan(ParFor *desc, uint32 idx)
MSpan *s;
USED(&desc);
- s = runtime·mheap->allspans[idx];
+ s = runtime·mheap.allspans[idx];
if(s->state != MSpanInUse)
return;
- arena_start = runtime·mheap->arena_start;
+ arena_start = runtime·mheap.arena_start;
p = (byte*)(s->start << PageShift);
cl = s->sizeclass;
size = s->elemsize;
@@ -1682,9 +1690,9 @@ sweepspan(ParFor *desc, uint32 idx)
// Free large span.
runtime·unmarkspan(p, 1<<PageShift);
*(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing
- runtime·MHeap_Free(runtime·mheap, s, 1);
- c->local_alloc -= size;
- c->local_nfree++;
+ runtime·MHeap_Free(&runtime·mheap, s, 1);
+ c->local_nlargefree++;
+ c->local_largefree += size;
} else {
// Free small object.
switch(compression) {
@@ -1705,12 +1713,9 @@ sweepspan(ParFor *desc, uint32 idx)
}
if(nfree) {
- c->local_by_size[cl].nfree += nfree;
- c->local_alloc -= size * nfree;
- c->local_nfree += nfree;
+ c->local_nsmallfree[cl] += nfree;
c->local_cachealloc -= nfree * size;
- c->local_objects -= nfree;
- runtime·MCentral_FreeSpan(&runtime·mheap->central[cl], s, nfree, head.next, end);
+ runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end);
}
}
@@ -1724,10 +1729,10 @@ dumpspan(uint32 idx)
MSpan *s;
bool allocated, special;
- s = runtime·mheap->allspans[idx];
+ s = runtime·mheap.allspans[idx];
if(s->state != MSpanInUse)
return;
- arena_start = runtime·mheap->arena_start;
+ arena_start = runtime·mheap.arena_start;
p = (byte*)(s->start << PageShift);
sizeclass = s->sizeclass;
size = s->elemsize;
@@ -1785,7 +1790,7 @@ runtime·memorydump(void)
{
uint32 spanidx;
- for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) {
+ for(spanidx=0; spanidx<runtime·mheap.nspan; spanidx++) {
dumpspan(spanidx);
}
}
@@ -1827,13 +1832,28 @@ runtime·gchelper(void)
static int32 gcpercent = GcpercentUnknown;
static void
-cachestats(GCStats *stats)
+cachestats(void)
+{
+ MCache *c;
+ P *p, **pp;
+
+ for(pp=runtime·allp; p=*pp; pp++) {
+ c = p->mcache;
+ if(c==nil)
+ continue;
+ runtime·purgecachedstats(c);
+ }
+}
+
+static void
+updatememstats(GCStats *stats)
{
M *mp;
+ MSpan *s;
MCache *c;
P *p, **pp;
int32 i;
- uint64 stacks_inuse;
+ uint64 stacks_inuse, smallfree;
uint64 *src, *dst;
if(stats)
@@ -1849,29 +1869,80 @@ cachestats(GCStats *stats)
runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
}
}
+ mstats.stacks_inuse = stacks_inuse;
+ mstats.mcache_inuse = runtime·mheap.cachealloc.inuse;
+ mstats.mspan_inuse = runtime·mheap.spanalloc.inuse;
+ mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys +
+ mstats.mcache_sys + mstats.buckhash_sys + mstats.gc_sys + mstats.other_sys;
+
+ // Calculate memory allocator stats.
+ // During program execution we only count number of frees and amount of freed memory.
+ // Current number of alive object in the heap and amount of alive heap memory
+ // are calculated by scanning all spans.
+ // Total number of mallocs is calculated as number of frees plus number of alive objects.
+ // Similarly, total amount of allocated memory is calculated as amount of freed memory
+ // plus amount of alive heap memory.
+ mstats.alloc = 0;
+ mstats.total_alloc = 0;
+ mstats.nmalloc = 0;
+ mstats.nfree = 0;
+ for(i = 0; i < nelem(mstats.by_size); i++) {
+ mstats.by_size[i].nmalloc = 0;
+ mstats.by_size[i].nfree = 0;
+ }
+
+ // Flush MCache's to MCentral.
for(pp=runtime·allp; p=*pp; pp++) {
c = p->mcache;
if(c==nil)
continue;
- runtime·purgecachedstats(c);
- for(i=0; i<nelem(c->local_by_size); i++) {
- mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc;
- c->local_by_size[i].nmalloc = 0;
- mstats.by_size[i].nfree += c->local_by_size[i].nfree;
- c->local_by_size[i].nfree = 0;
+ runtime·MCache_ReleaseAll(c);
+ }
+
+ // Aggregate local stats.
+ cachestats();
+
+ // Scan all spans and count number of alive objects.
+ for(i = 0; i < runtime·mheap.nspan; i++) {
+ s = runtime·mheap.allspans[i];
+ if(s->state != MSpanInUse)
+ continue;
+ if(s->sizeclass == 0) {
+ mstats.nmalloc++;
+ mstats.alloc += s->elemsize;
+ } else {
+ mstats.nmalloc += s->ref;
+ mstats.by_size[s->sizeclass].nmalloc += s->ref;
+ mstats.alloc += s->ref*s->elemsize;
}
}
- mstats.stacks_inuse = stacks_inuse;
+
+ // Aggregate by size class.
+ smallfree = 0;
+ mstats.nfree = runtime·mheap.nlargefree;
+ for(i = 0; i < nelem(mstats.by_size); i++) {
+ mstats.nfree += runtime·mheap.nsmallfree[i];
+ mstats.by_size[i].nfree = runtime·mheap.nsmallfree[i];
+ mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i];
+ smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i];
+ }
+ mstats.nmalloc += mstats.nfree;
+
+ // Calculate derived stats.
+ mstats.total_alloc = mstats.alloc + runtime·mheap.largefree + smallfree;
+ mstats.heap_alloc = mstats.alloc;
+ mstats.heap_objects = mstats.nmalloc - mstats.nfree;
}
// Structure of arguments passed to function gc().
-// This allows the arguments to be passed via reflect·call.
+// This allows the arguments to be passed via runtime·mcall.
struct gc_args
{
- int32 force;
+ int64 start_time; // start time of GC in ns (just before stoptheworld)
};
static void gc(struct gc_args *args);
+static void mgc(G *gp);
static int32
readgogc(void)
@@ -1886,12 +1957,13 @@ readgogc(void)
return runtime·atoi(p);
}
+static FuncVal runfinqv = {runfinq};
+
void
runtime·gc(int32 force)
{
- byte *p;
- struct gc_args a, *ap;
- FuncVal gcv;
+ struct gc_args a;
+ int32 i;
// The atomic operations are not atomic if the uint64s
// are not aligned on uint64 boundaries. This has been
@@ -1909,34 +1981,77 @@ runtime·gc(int32 force)
// problems, don't bother trying to run gc
// while holding a lock. The next mallocgc
// without a lock will do the gc instead.
- if(!mstats.enablegc || m->locks > 0 || runtime·panicking)
+ if(!mstats.enablegc || g == m->g0 || m->locks > 0 || runtime·panicking)
return;
if(gcpercent == GcpercentUnknown) { // first time through
- gcpercent = readgogc();
-
- p = runtime·getenv("GOGCTRACE");
- if(p != nil)
- gctrace = runtime·atoi(p);
+ runtime·lock(&runtime·mheap);
+ if(gcpercent == GcpercentUnknown)
+ gcpercent = readgogc();
+ runtime·unlock(&runtime·mheap);
}
if(gcpercent < 0)
return;
- // Run gc on a bigger stack to eliminate
- // a potentially large number of calls to runtime·morestack.
- a.force = force;
- ap = &a;
- m->moreframesize_minalloc = StackBig;
- gcv.fn = (void*)gc;
- reflect·call(&gcv, (byte*)&ap, sizeof(ap));
-
- if(gctrace > 1 && !force) {
- a.force = 1;
- gc(&a);
+ runtime·semacquire(&runtime·worldsema, false);
+ if(!force && mstats.heap_alloc < mstats.next_gc) {
+ // typically threads which lost the race to grab
+ // worldsema exit here when gc is done.
+ runtime·semrelease(&runtime·worldsema);
+ return;
}
+
+ // Ok, we're doing it! Stop everybody else
+ a.start_time = runtime·nanotime();
+ m->gcing = 1;
+ runtime·stoptheworld();
+
+ // Run gc on the g0 stack. We do this so that the g stack
+ // we're currently running on will no longer change. Cuts
+ // the root set down a bit (g0 stacks are not scanned, and
+ // we don't need to scan gc's internal state). Also an
+ // enabler for copyable stacks.
+ for(i = 0; i < (runtime·debug.gctrace > 1 ? 2 : 1); i++) {
+ // switch to g0, call gc(&a), then switch back
+ g->param = &a;
+ g->status = Gwaiting;
+ g->waitreason = "garbage collection";
+ runtime·mcall(mgc);
+ // record a new start time in case we're going around again
+ a.start_time = runtime·nanotime();
+ }
+
+ // all done
+ m->gcing = 0;
+ m->locks++;
+ runtime·semrelease(&runtime·worldsema);
+ runtime·starttheworld();
+ m->locks--;
+
+ // now that gc is done, kick off finalizer thread if needed
+ if(finq != nil) {
+ runtime·lock(&finlock);
+ // kick off or wake up goroutine to run queued finalizers
+ if(fing == nil)
+ fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
+ else if(fingwait) {
+ fingwait = 0;
+ runtime·ready(fing);
+ }
+ runtime·unlock(&finlock);
+ }
+ // give the queued finalizers, if any, a chance to run
+ runtime·gosched();
}
-static FuncVal runfinqv = {runfinq};
+static void
+mgc(G *gp)
+{
+ gc(gp->param);
+ gp->param = nil;
+ gp->status = Grunning;
+ runtime·gogo(&gp->sched);
+}
static void
gc(struct gc_args *args)
@@ -1948,27 +2063,18 @@ gc(struct gc_args *args)
uint32 i;
Eface eface;
- runtime·semacquire(&runtime·worldsema);
- if(!args->force && mstats.heap_alloc < mstats.next_gc) {
- runtime·semrelease(&runtime·worldsema);
- return;
- }
-
- t0 = runtime·nanotime();
-
- m->gcing = 1;
- runtime·stoptheworld();
+ t0 = args->start_time;
if(CollectStats)
runtime·memclr((byte*)&gcstats, sizeof(gcstats));
for(mp=runtime·allm; mp; mp=mp->alllink)
- runtime·settype_flush(mp, false);
+ runtime·settype_flush(mp);
heap0 = 0;
obj0 = 0;
- if(gctrace) {
- cachestats(nil);
+ if(runtime·debug.gctrace) {
+ updatememstats(nil);
heap0 = mstats.heap_alloc;
obj0 = mstats.nmalloc - mstats.nfree;
}
@@ -1992,7 +2098,7 @@ gc(struct gc_args *args)
work.nproc = runtime·gcprocs();
addroots();
runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
- runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap->nspan, nil, true, sweepspan);
+ runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan);
if(work.nproc > 1) {
runtime·noteclear(&work.alldone);
runtime·helpgc(work.nproc);
@@ -2018,29 +2124,8 @@ gc(struct gc_args *args)
if(work.nproc > 1)
runtime·notesleep(&work.alldone);
- cachestats(&stats);
-
- stats.nprocyield += work.sweepfor->nprocyield;
- stats.nosyield += work.sweepfor->nosyield;
- stats.nsleep += work.sweepfor->nsleep;
-
+ cachestats();
mstats.next_gc = mstats.heap_alloc+mstats.heap_alloc*gcpercent/100;
- m->gcing = 0;
-
- if(finq != nil) {
- m->locks++; // disable gc during the mallocs in newproc
- // kick off or wake up goroutine to run queued finalizers
- if(fing == nil)
- fing = runtime·newproc1(&runfinqv, nil, 0, 0, runtime·gc);
- else if(fingwait) {
- fingwait = 0;
- runtime·ready(fing);
- }
- m->locks--;
- }
-
- heap1 = mstats.heap_alloc;
- obj1 = mstats.nmalloc - mstats.nfree;
t4 = runtime·nanotime();
mstats.last_gc = t4;
@@ -2050,7 +2135,15 @@ gc(struct gc_args *args)
if(mstats.debuggc)
runtime·printf("pause %D\n", t4-t0);
- if(gctrace) {
+ if(runtime·debug.gctrace) {
+ updatememstats(&stats);
+ heap1 = mstats.heap_alloc;
+ obj1 = mstats.nmalloc - mstats.nfree;
+
+ stats.nprocyield += work.sweepfor->nprocyield;
+ stats.nosyield += work.sweepfor->nosyield;
+ stats.nsleep += work.sweepfor->nsleep;
+
runtime·printf("gc%d(%d): %D+%D+%D ms, %D -> %D MB %D -> %D (%D-%D) objects,"
" %D(%D) handoff, %D(%D) steal, %D/%D/%D yields\n",
mstats.numgc, work.nproc, (t2-t1)/1000000, (t3-t2)/1000000, (t1-t0+t4-t3)/1000000,
@@ -2079,16 +2172,13 @@ gc(struct gc_args *args)
runtime·printf("\ttotal:\t%D\n", ninstr);
runtime·printf("putempty: %D, getfull: %D\n", gcstats.putempty, gcstats.getfull);
+
+ runtime·printf("markonly base lookup: bit %D word %D span %D\n", gcstats.markonly.foundbit, gcstats.markonly.foundword, gcstats.markonly.foundspan);
+ runtime·printf("flushptrbuf base lookup: bit %D word %D span %D\n", gcstats.flushptrbuf.foundbit, gcstats.flushptrbuf.foundword, gcstats.flushptrbuf.foundspan);
}
}
runtime·MProf_GC();
- runtime·semrelease(&runtime·worldsema);
- runtime·starttheworld();
-
- // give the queued finalizers, if any, a chance to run
- if(finq != nil)
- runtime·gosched();
}
void
@@ -2098,14 +2188,16 @@ runtime·ReadMemStats(MStats *stats)
// because stoptheworld can only be used by
// one goroutine at a time, and there might be
// a pending garbage collection already calling it.
- runtime·semacquire(&runtime·worldsema);
+ runtime·semacquire(&runtime·worldsema, false);
m->gcing = 1;
runtime·stoptheworld();
- cachestats(nil);
+ updatememstats(nil);
*stats = mstats;
m->gcing = 0;
+ m->locks++;
runtime·semrelease(&runtime·worldsema);
runtime·starttheworld();
+ m->locks--;
}
void
@@ -2120,7 +2212,7 @@ runtime∕debug·readGCStats(Slice *pauses)
// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
p = (uint64*)pauses->array;
- runtime·lock(runtime·mheap);
+ runtime·lock(&runtime·mheap);
n = mstats.numgc;
if(n > nelem(mstats.pause_ns))
n = nelem(mstats.pause_ns);
@@ -2135,21 +2227,21 @@ runtime∕debug·readGCStats(Slice *pauses)
p[n] = mstats.last_gc;
p[n+1] = mstats.numgc;
p[n+2] = mstats.pause_total_ns;
- runtime·unlock(runtime·mheap);
+ runtime·unlock(&runtime·mheap);
pauses->len = n+3;
}
void
runtime∕debug·setGCPercent(intgo in, intgo out)
{
- runtime·lock(runtime·mheap);
+ runtime·lock(&runtime·mheap);
if(gcpercent == GcpercentUnknown)
gcpercent = readgogc();
out = gcpercent;
if(in < 0)
in = -1;
gcpercent = in;
- runtime·unlock(runtime·mheap);
+ runtime·unlock(&runtime·mheap);
FLUSH(&out);
}
@@ -2160,6 +2252,8 @@ gchelperstart(void)
runtime·throw("gchelperstart: bad m->helpgc");
if(runtime·xchg(&bufferList[m->helpgc].busy, 1))
runtime·throw("gchelperstart: already busy");
+ if(g != m->g0)
+ runtime·throw("gchelper not running on g0 stack");
}
static void
@@ -2169,39 +2263,57 @@ runfinq(void)
FinBlock *fb, *next;
byte *frame;
uint32 framesz, framecap, i;
+ Eface *ef, ef1;
frame = nil;
framecap = 0;
for(;;) {
- // There's no need for a lock in this section
- // because it only conflicts with the garbage
- // collector, and the garbage collector only
- // runs when everyone else is stopped, and
- // runfinq only stops at the gosched() or
- // during the calls in the for loop.
+ runtime·lock(&finlock);
fb = finq;
finq = nil;
if(fb == nil) {
fingwait = 1;
- runtime·park(nil, nil, "finalizer wait");
+ runtime·park(runtime·unlock, &finlock, "finalizer wait");
continue;
}
+ runtime·unlock(&finlock);
if(raceenabled)
runtime·racefingo();
for(; fb; fb=next) {
next = fb->next;
for(i=0; i<fb->cnt; i++) {
f = &fb->fin[i];
- framesz = sizeof(uintptr) + f->nret;
+ framesz = sizeof(Eface) + f->nret;
if(framecap < framesz) {
runtime·free(frame);
- frame = runtime·mal(framesz);
+ // The frame does not contain pointers interesting for GC,
+ // all not yet finalized objects are stored in finc.
+ // If we do not mark it as FlagNoScan,
+ // the last finalized object is not collected.
+ frame = runtime·mallocgc(framesz, 0, FlagNoScan|FlagNoInvokeGC);
framecap = framesz;
}
- *(void**)frame = f->arg;
- reflect·call(f->fn, frame, sizeof(uintptr) + f->nret);
+ if(f->fint == nil)
+ runtime·throw("missing type in runfinq");
+ if(f->fint->kind == KindPtr) {
+ // direct use of pointer
+ *(void**)frame = f->arg;
+ } else if(((InterfaceType*)f->fint)->mhdr.len == 0) {
+ // convert to empty interface
+ ef = (Eface*)frame;
+ ef->type = f->ot;
+ ef->data = f->arg;
+ } else {
+ // convert to interface with methods, via empty interface.
+ ef1.type = f->ot;
+ ef1.data = f->arg;
+ if(!runtime·ifaceE2I2((InterfaceType*)f->fint, ef1, (Iface*)frame))
+ runtime·throw("invalid type conversion in runfinq");
+ }
+ reflect·call(f->fn, frame, framesz);
f->fn = nil;
f->arg = nil;
+ f->ot = nil;
}
fb->cnt = 0;
fb->next = finc;
@@ -2212,28 +2324,28 @@ runfinq(void)
}
// mark the block at v of size n as allocated.
-// If noptr is true, mark it as having no pointers.
+// If noscan is true, mark it as not needing scanning.
void
-runtime·markallocated(void *v, uintptr n, bool noptr)
+runtime·markallocated(void *v, uintptr n, bool noscan)
{
uintptr *b, obits, bits, off, shift;
if(0)
runtime·printf("markallocated %p+%p\n", v, n);
- if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
+ if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markallocated: bad pointer");
- off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offset
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
for(;;) {
obits = *b;
bits = (obits & ~(bitMask<<shift)) | (bitAllocated<<shift);
- if(noptr)
- bits |= bitNoPointers<<shift;
- if(runtime·singleproc) {
+ if(noscan)
+ bits |= bitNoScan<<shift;
+ if(runtime·gomaxprocs == 1) {
*b = bits;
break;
} else {
@@ -2251,19 +2363,19 @@ runtime·markfreed(void *v, uintptr n)
uintptr *b, obits, bits, off, shift;
if(0)
- runtime·printf("markallocated %p+%p\n", v, n);
+ runtime·printf("markfreed %p+%p\n", v, n);
- if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
- runtime·throw("markallocated: bad pointer");
+ if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
+ runtime·throw("markfreed: bad pointer");
- off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offset
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
for(;;) {
obits = *b;
bits = (obits & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
- if(runtime·singleproc) {
+ if(runtime·gomaxprocs == 1) {
*b = bits;
break;
} else {
@@ -2283,11 +2395,11 @@ runtime·checkfreed(void *v, uintptr n)
if(!runtime·checking)
return;
- if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
+ if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
return; // not allocated, so okay
- off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start; // word offset
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start; // word offset
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
bits = *b>>shift;
@@ -2306,7 +2418,7 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
uintptr *b, off, shift;
byte *p;
- if((byte*)v+size*n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
+ if((byte*)v+size*n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markspan: bad pointer");
p = v;
@@ -2317,8 +2429,8 @@ runtime·markspan(void *v, uintptr size, uintptr n, bool leftover)
// the entire span, and each bitmap word has bits for only
// one span, so no other goroutines are changing these
// bitmap words.
- off = (uintptr*)p - (uintptr*)runtime·mheap->arena_start; // word offset
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)p - (uintptr*)runtime·mheap.arena_start; // word offset
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
*b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
}
@@ -2330,14 +2442,14 @@ runtime·unmarkspan(void *v, uintptr n)
{
uintptr *p, *b, off;
- if((byte*)v+n > (byte*)runtime·mheap->arena_used || (byte*)v < runtime·mheap->arena_start)
+ if((byte*)v+n > (byte*)runtime·mheap.arena_used || (byte*)v < runtime·mheap.arena_start)
runtime·throw("markspan: bad pointer");
p = v;
- off = p - (uintptr*)runtime·mheap->arena_start; // word offset
+ off = p - (uintptr*)runtime·mheap.arena_start; // word offset
if(off % wordsPerBitmapWord != 0)
runtime·throw("markspan: unaligned pointer");
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
n /= PtrSize;
if(n%wordsPerBitmapWord != 0)
runtime·throw("unmarkspan: unaligned length");
@@ -2358,8 +2470,8 @@ runtime·blockspecial(void *v)
if(DebugMark)
return true;
- off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start;
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
return (*b & (bitSpecial<<shift)) != 0;
@@ -2373,8 +2485,8 @@ runtime·setblockspecial(void *v, bool s)
if(DebugMark)
return;
- off = (uintptr*)v - (uintptr*)runtime·mheap->arena_start;
- b = (uintptr*)runtime·mheap->arena_start - off/wordsPerBitmapWord - 1;
+ off = (uintptr*)v - (uintptr*)runtime·mheap.arena_start;
+ b = (uintptr*)runtime·mheap.arena_start - off/wordsPerBitmapWord - 1;
shift = off % wordsPerBitmapWord;
for(;;) {
@@ -2383,7 +2495,7 @@ runtime·setblockspecial(void *v, bool s)
bits = obits | (bitSpecial<<shift);
else
bits = obits & ~(bitSpecial<<shift);
- if(runtime·singleproc) {
+ if(runtime·gomaxprocs == 1) {
*b = bits;
break;
} else {
@@ -2406,10 +2518,10 @@ runtime·MHeap_MapBits(MHeap *h)
uintptr n;
n = (h->arena_used - h->arena_start) / wordsPerBitmapWord;
- n = (n+bitmapChunk-1) & ~(bitmapChunk-1);
+ n = ROUND(n, bitmapChunk);
if(h->bitmap_mapped >= n)
return;
- runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped);
+ runtime·SysMap(h->arena_start - n, n - h->bitmap_mapped, &mstats.gc_sys);
h->bitmap_mapped = n;
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-16 03:33:08 +0000