summaryrefslogtreecommitdiff
path: root/src/pkg/runtime/mgc0.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/runtime/mgc0.c')
-rw-r--r--src/pkg/runtime/mgc0.c396
1 files changed, 273 insertions, 123 deletions
diff --git a/src/pkg/runtime/mgc0.c b/src/pkg/runtime/mgc0.c
index 010f9cd96..f9dbdbb4a 100644
--- a/src/pkg/runtime/mgc0.c
+++ b/src/pkg/runtime/mgc0.c
@@ -18,6 +18,8 @@ enum {
Debug = 0,
DebugMark = 0, // run second pass to check mark
CollectStats = 0,
+ ScanStackByFrames = 0,
+ IgnorePreciseGC = 0,
// Four bits per word (see #defines below).
wordsPerBitmapWord = sizeof(void*)*8/4,
@@ -140,6 +142,7 @@ static Workbuf* getempty(Workbuf*);
static Workbuf* getfull(Workbuf*);
static void putempty(Workbuf*);
static Workbuf* handoff(Workbuf*);
+static void gchelperstart(void);
static struct {
uint64 full; // lock-free list of full blocks
@@ -191,7 +194,7 @@ static struct {
// markonly marks an object. It returns true if the object
// has been marked by this function, false otherwise.
-// This function isn't thread-safe and doesn't append the object to any buffer.
+// This function doesn't append the object to any buffer.
static bool
markonly(void *obj)
{
@@ -254,13 +257,23 @@ found:
// Only care about allocated and not marked.
if((bits & (bitAllocated|bitMarked)) != bitAllocated)
return false;
- *bitp |= bitMarked<<shift;
+ if(work.nproc == 1)
+ *bitp |= bitMarked<<shift;
+ else {
+ for(;;) {
+ x = *bitp;
+ if(x & (bitMarked<<shift))
+ return false;
+ if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+ break;
+ }
+ }
// The object is now marked
return true;
}
-// PtrTarget and BitTarget are structures used by intermediate buffers.
+// PtrTarget is a structure used by intermediate buffers.
// The intermediate buffers hold GC data before it
// is moved/flushed to the work buffer (Workbuf).
// The size of an intermediate buffer is very small,
@@ -272,25 +285,17 @@ struct PtrTarget
uintptr ti;
};
-typedef struct BitTarget BitTarget;
-struct BitTarget
-{
- void *p;
- uintptr ti;
- uintptr *bitp, shift;
-};
-
typedef struct BufferList BufferList;
struct BufferList
{
PtrTarget ptrtarget[IntermediateBufferCapacity];
- BitTarget bittarget[IntermediateBufferCapacity];
Obj obj[IntermediateBufferCapacity];
- BufferList *next;
+ uint32 busy;
+ byte pad[CacheLineSize];
};
-static BufferList *bufferList;
+#pragma dataflag 16 // no pointers
+static BufferList bufferList[MaxGcproc];
-static Lock lock;
static Type *itabtype;
static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
@@ -301,7 +306,6 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
// and are prepared to be scanned by the garbage collector.
//
// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
-// bitbuf holds temporary data generated by this function.
//
// A simplified drawing explaining how the todo-list moves from a structure to another:
//
@@ -309,14 +313,12 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
// (find pointers)
// Obj ------> PtrTarget (pointer targets)
// ↑ |
-// | | flushptrbuf (1st part,
-// | | find block start)
-// | ↓
-// `--------- BitTarget (pointer targets and the corresponding locations in bitmap)
-// flushptrbuf
-// (2nd part, mark and enqueue)
+// | |
+// `----------'
+// flushptrbuf
+// (find block start, mark and enqueue)
static void
-flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, BitTarget *bitbuf)
+flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj)
{
byte *p, *arena_start, *obj;
uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n;
@@ -325,7 +327,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
Obj *wp;
Workbuf *wbuf;
PtrTarget *ptrbuf_end;
- BitTarget *bitbufpos, *bt;
arena_start = runtime·mheap->arena_start;
@@ -359,8 +360,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
{
// Multi-threaded version.
- bitbufpos = bitbuf;
-
while(ptrbuf < ptrbuf_end) {
obj = ptrbuf->p;
ti = ptrbuf->ti;
@@ -438,26 +437,22 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
// Only care about allocated and not marked.
if((bits & (bitAllocated|bitMarked)) != bitAllocated)
continue;
-
- *bitbufpos++ = (BitTarget){obj, ti, bitp, shift};
- }
-
- runtime·lock(&lock);
- for(bt=bitbuf; bt<bitbufpos; bt++){
- xbits = *bt->bitp;
- bits = xbits >> bt->shift;
- if((bits & bitMarked) != 0)
- continue;
-
- // Mark the block
- *bt->bitp = xbits | (bitMarked << bt->shift);
+ if(work.nproc == 1)
+ *bitp |= bitMarked<<shift;
+ else {
+ for(;;) {
+ x = *bitp;
+ if(x & (bitMarked<<shift))
+ goto continue_obj;
+ if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+ break;
+ }
+ }
// If object has no pointers, don't need to scan further.
if((bits & bitNoPointers) != 0)
continue;
- obj = bt->p;
-
// Ask span about size class.
// (Manually inlined copy of MHeap_Lookup.)
x = (uintptr)obj >> PageShift;
@@ -467,11 +462,11 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
PREFETCH(obj);
- *wp = (Obj){obj, s->elemsize, bt->ti};
+ *wp = (Obj){obj, s->elemsize, ti};
wp++;
nobj++;
+ continue_obj:;
}
- runtime·unlock(&lock);
// If another proc wants a pointer, give it some.
if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
@@ -559,6 +554,59 @@ struct Frame {
uintptr *loop_or_ret;
};
+// Sanity check for the derived type info objti.
+static void
+checkptr(void *obj, uintptr objti)
+{
+ uintptr *pc1, *pc2, type, tisize, i, j, x;
+ byte *objstart;
+ Type *t;
+ MSpan *s;
+
+ if(!Debug)
+ runtime·throw("checkptr is debug only");
+
+ if(obj < runtime·mheap->arena_start || obj >= runtime·mheap->arena_used)
+ return;
+ type = runtime·gettype(obj);
+ t = (Type*)(type & ~(uintptr)(PtrSize-1));
+ if(t == nil)
+ return;
+ x = (uintptr)obj >> PageShift;
+ if(sizeof(void*) == 8)
+ x -= (uintptr)(runtime·mheap->arena_start)>>PageShift;
+ s = runtime·mheap->map[x];
+ objstart = (byte*)((uintptr)s->start<<PageShift);
+ if(s->sizeclass != 0) {
+ i = ((byte*)obj - objstart)/s->elemsize;
+ objstart += i*s->elemsize;
+ }
+ tisize = *(uintptr*)objti;
+ // Sanity check for object size: it should fit into the memory block.
+ if((byte*)obj + tisize > objstart + s->elemsize)
+ runtime·throw("invalid gc type info");
+ if(obj != objstart)
+ return;
+ // If obj points to the beginning of the memory block,
+ // check type info as well.
+ if(t->string == nil ||
+ // Gob allocates unsafe pointers for indirection.
+ (runtime·strcmp(t->string->str, (byte*)"unsafe.Pointer") &&
+ // Runtime and gc think differently about closures.
+ runtime·strstr(t->string->str, (byte*)"struct { F uintptr") != t->string->str)) {
+ pc1 = (uintptr*)objti;
+ pc2 = (uintptr*)t->gc;
+ // A simple best-effort check until first GC_END.
+ for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) {
+ if(pc1[j] != pc2[j]) {
+ runtime·printf("invalid gc type info for '%s' at %p, type info %p, block info %p\n",
+ t->string ? (int8*)t->string->str : (int8*)"?", j, pc1[j], pc2[j]);
+ runtime·throw("invalid gc type info");
+ }
+ }
+ }
+}
+
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
@@ -575,20 +623,19 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
byte *b, *arena_start, *arena_used;
uintptr n, i, end_b, elemsize, size, ti, objti, count, type;
uintptr *pc, precise_type, nominal_size;
- uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti;
+ uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ret;
void *obj;
Type *t;
Slice *sliceptr;
Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
BufferList *scanbuffers;
PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos;
- BitTarget *bitbuf;
Obj *objbuf, *objbuf_end, *objbufpos;
Eface *eface;
Iface *iface;
Hmap *hmap;
MapType *maptype;
- bool didmark, mapkey_kind, mapval_kind;
+ bool mapkey_kind, mapval_kind;
struct hash_gciter map_iter;
struct hash_gciter_data d;
Hchan *chan;
@@ -606,26 +653,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
precise_type = false;
nominal_size = 0;
- // Allocate ptrbuf, bitbuf
+ // Allocate ptrbuf
{
- runtime·lock(&lock);
-
- if(bufferList == nil) {
- bufferList = runtime·SysAlloc(sizeof(*bufferList));
- if(bufferList == nil)
- runtime·throw("runtime: cannot allocate memory");
- bufferList->next = nil;
- }
- scanbuffers = bufferList;
- bufferList = bufferList->next;
-
+ scanbuffers = &bufferList[m->helpgc];
ptrbuf = &scanbuffers->ptrtarget[0];
ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget);
- bitbuf = &scanbuffers->bittarget[0];
objbuf = &scanbuffers->obj[0];
objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj);
-
- runtime·unlock(&lock);
}
ptrbufpos = ptrbuf;
@@ -638,6 +672,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
mapkey_ti = mapval_ti = 0;
chan = nil;
chantype = nil;
+ chan_ret = nil;
goto next_block;
@@ -666,6 +701,17 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
} else {
stack_top.count = 1;
}
+ if(Debug) {
+ // Simple sanity check for provided type info ti:
+ // The declared size of the object must be not larger than the actual size
+ // (it can be smaller due to inferior pointers).
+ // It's difficult to make a comprehensive check due to inferior pointers,
+ // reflection, gob, etc.
+ if(pc[0] > n) {
+ runtime·printf("invalid gc type info: type info size %p, block size %p\n", pc[0], n);
+ runtime·throw("invalid gc type info");
+ }
+ }
} else if(UseSpanType) {
if(CollectStats)
runtime·xadd64(&gcstats.obj.notype, 1);
@@ -703,7 +749,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
mapval_kind = maptype->elem->kind;
mapval_ti = (uintptr)maptype->elem->gc | PRECISE;
- map_ret = 0;
+ map_ret = nil;
pc = mapProg;
} else {
goto next_block;
@@ -712,6 +758,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
case TypeInfo_Chan:
chan = (Hchan*)b;
chantype = (ChanType*)t;
+ chan_ret = nil;
pc = chanProg;
break;
default:
@@ -725,6 +772,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
pc = defaultProg;
}
+ if(IgnorePreciseGC)
+ pc = defaultProg;
+
pc++;
stack_top.b = (uintptr)b;
@@ -741,6 +791,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
obj = *(void**)(stack_top.b + pc[1]);
objti = pc[2];
pc += 3;
+ if(Debug)
+ checkptr(obj, objti);
break;
case GC_SLICE:
@@ -759,17 +811,29 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
case GC_STRING:
obj = *(void**)(stack_top.b + pc[1]);
+ markonly(obj);
pc += 2;
- break;
+ continue;
case GC_EFACE:
eface = (Eface*)(stack_top.b + pc[1]);
pc += 2;
- if(eface->type != nil && (eface->data >= arena_start && eface->data < arena_used)) {
- t = eface->type;
+ if(eface->type == nil)
+ continue;
+
+ // eface->type
+ t = eface->type;
+ if((void*)t >= arena_start && (void*)t < arena_used) {
+ *ptrbufpos++ = (PtrTarget){t, 0};
+ if(ptrbufpos == ptrbuf_end)
+ flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+ }
+
+ // eface->data
+ if(eface->data >= arena_start && eface->data < arena_used) {
if(t->size <= sizeof(void*)) {
if((t->kind & KindNoPointers))
- break;
+ continue;
obj = eface->data;
if((t->kind & ~KindNoPointers) == KindPtr)
@@ -785,13 +849,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
iface = (Iface*)(stack_top.b + pc[1]);
pc += 2;
if(iface->tab == nil)
- break;
+ continue;
// iface->tab
if((void*)iface->tab >= arena_start && (void*)iface->tab < arena_used) {
*ptrbufpos++ = (PtrTarget){iface->tab, (uintptr)itabtype->gc};
if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+ flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
}
// iface->data
@@ -799,7 +863,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
t = iface->tab->type;
if(t->size <= sizeof(void*)) {
if((t->kind & KindNoPointers))
- break;
+ continue;
obj = iface->data;
if((t->kind & ~KindNoPointers) == KindPtr)
@@ -812,13 +876,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
break;
case GC_DEFAULT_PTR:
- while((i = stack_top.b) <= end_b) {
+ while(stack_top.b <= end_b) {
+ obj = *(byte**)stack_top.b;
stack_top.b += PtrSize;
- obj = *(byte**)i;
if(obj >= arena_start && obj < arena_used) {
*ptrbufpos++ = (PtrTarget){obj, 0};
if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+ flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
}
}
goto next_block;
@@ -826,9 +890,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
case GC_END:
if(--stack_top.count != 0) {
// Next iteration of a loop if possible.
- elemsize = stack_top.elemsize;
- stack_top.b += elemsize;
- if(stack_top.b + elemsize <= end_b+PtrSize) {
+ stack_top.b += stack_top.elemsize;
+ if(stack_top.b + stack_top.elemsize <= end_b+PtrSize) {
pc = stack_top.loop_or_ret;
continue;
}
@@ -894,10 +957,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
pc += 3;
continue;
}
- runtime·lock(&lock);
- didmark = markonly(hmap);
- runtime·unlock(&lock);
- if(didmark) {
+ if(markonly(hmap)) {
maptype = (MapType*)pc[2];
if(hash_gciter_init(hmap, &map_iter)) {
mapkey_size = maptype->key->size;
@@ -923,31 +983,41 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
while(hash_gciter_next(&map_iter, &d)) {
// buffers: reserve space for 2 objects.
if(ptrbufpos+2 >= ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+ flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
if(objbufpos+2 >= objbuf_end)
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
- if(d.st != nil) {
- runtime·lock(&lock);
+ if(d.st != nil)
markonly(d.st);
- runtime·unlock(&lock);
- }
+
if(d.key_data != nil) {
if(!(mapkey_kind & KindNoPointers) || d.indirectkey) {
if(!d.indirectkey)
*objbufpos++ = (Obj){d.key_data, mapkey_size, mapkey_ti};
- else
+ else {
+ if(Debug) {
+ obj = *(void**)d.key_data;
+ if(!(arena_start <= obj && obj < arena_used))
+ runtime·throw("scanblock: inconsistent hashmap");
+ }
*ptrbufpos++ = (PtrTarget){*(void**)d.key_data, mapkey_ti};
+ }
}
if(!(mapval_kind & KindNoPointers) || d.indirectval) {
if(!d.indirectval)
*objbufpos++ = (Obj){d.val_data, mapval_size, mapval_ti};
- else
+ else {
+ if(Debug) {
+ obj = *(void**)d.val_data;
+ if(!(arena_start <= obj && obj < arena_used))
+ runtime·throw("scanblock: inconsistent hashmap");
+ }
*ptrbufpos++ = (PtrTarget){*(void**)d.val_data, mapval_ti};
+ }
}
}
}
- if(map_ret == 0)
+ if(map_ret == nil)
goto next_block;
pc = map_ret;
continue;
@@ -961,7 +1031,26 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
*objbufpos++ = (Obj){obj, size, objti};
if(objbufpos == objbuf_end)
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
- break;
+ continue;
+
+ case GC_CHAN_PTR:
+ // Similar to GC_MAP_PTR
+ chan = *(Hchan**)(stack_top.b + pc[1]);
+ if(chan == nil) {
+ pc += 3;
+ continue;
+ }
+ if(markonly(chan)) {
+ chantype = (ChanType*)pc[2];
+ if(!(chantype->elem->kind & KindNoPointers)) {
+ // Start chanProg.
+ chan_ret = pc+3;
+ pc = chanProg+1;
+ continue;
+ }
+ }
+ pc += 3;
+ continue;
case GC_CHAN:
// There are no heap pointers in struct Hchan,
@@ -981,7 +1070,10 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
}
}
- goto next_block;
+ if(chan_ret == nil)
+ goto next_block;
+ pc = chan_ret;
+ continue;
default:
runtime·throw("scanblock: invalid GC instruction");
@@ -991,7 +1083,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
if(obj >= arena_start && obj < arena_used) {
*ptrbufpos++ = (PtrTarget){obj, objti};
if(ptrbufpos == ptrbuf_end)
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+ flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
}
}
@@ -1000,7 +1092,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
// the loop by setting b, n, ti to the parameters for the next block.
if(nobj == 0) {
- flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+ flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
if(nobj == 0) {
@@ -1026,11 +1118,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
nobj--;
}
-endscan:
- runtime·lock(&lock);
- scanbuffers->next = bufferList;
- bufferList = scanbuffers;
- runtime·unlock(&lock);
+endscan:;
}
// debug_scanblock is the debug copy of scanblock.
@@ -1299,51 +1387,94 @@ addroot(Obj obj)
work.nroot++;
}
+// Scan a stack frame. The doframe parameter is a signal that the previously
+// scanned activation has an unknown argument size. When *doframe is true the
+// current activation must have its entire frame scanned. Otherwise, only the
+// locals need to be scanned.
+static void
+addframeroots(Func *f, byte*, byte *sp, void *doframe)
+{
+ uintptr outs;
+
+ if(thechar == '5')
+ sp += sizeof(uintptr);
+ if(f->locals == 0 || *(bool*)doframe == true)
+ addroot((Obj){sp, f->frame - sizeof(uintptr), 0});
+ else if(f->locals > 0) {
+ outs = f->frame - sizeof(uintptr) - f->locals;
+ addroot((Obj){sp + outs, f->locals, 0});
+ }
+ if(f->args > 0)
+ addroot((Obj){sp + f->frame, f->args, 0});
+ *(bool*)doframe = (f->args == ArgsSizeUnknown);
+}
+
static void
addstackroots(G *gp)
{
M *mp;
int32 n;
Stktop *stk;
- byte *sp, *guard;
+ byte *sp, *guard, *pc;
+ Func *f;
+ bool doframe;
stk = (Stktop*)gp->stackbase;
guard = (byte*)gp->stackguard;
if(gp == g) {
// Scanning our own stack: start at &gp.
- sp = (byte*)&gp;
+ sp = runtime·getcallersp(&gp);
+ pc = runtime·getcallerpc(&gp);
} else if((mp = gp->m) != nil && mp->helpgc) {
// gchelper's stack is in active use and has no interesting pointers.
return;
+ } else if(gp->gcstack != (uintptr)nil) {
+ // Scanning another goroutine that is about to enter or might
+ // have just exited a system call. It may be executing code such
+ // as schedlock and may have needed to start a new stack segment.
+ // Use the stack segment and stack pointer at the time of
+ // the system call instead, since that won't change underfoot.
+ sp = (byte*)gp->gcsp;
+ pc = gp->gcpc;
+ stk = (Stktop*)gp->gcstack;
+ guard = (byte*)gp->gcguard;
} else {
// Scanning another goroutine's stack.
// The goroutine is usually asleep (the world is stopped).
sp = (byte*)gp->sched.sp;
-
- // The exception is that if the goroutine is about to enter or might
- // have just exited a system call, it may be executing code such
- // as schedlock and may have needed to start a new stack segment.
- // Use the stack segment and stack pointer at the time of
- // the system call instead, since that won't change underfoot.
- if(gp->gcstack != (uintptr)nil) {
- stk = (Stktop*)gp->gcstack;
- sp = (byte*)gp->gcsp;
- guard = (byte*)gp->gcguard;
+ pc = gp->sched.pc;
+ if(ScanStackByFrames && pc == (byte*)runtime·goexit && gp->fnstart != nil) {
+ // The goroutine has not started. However, its incoming
+ // arguments are live at the top of the stack and must
+ // be scanned. No other live values should be on the
+ // stack.
+ f = runtime·findfunc((uintptr)gp->fnstart->fn);
+ if(f->args > 0) {
+ if(thechar == '5')
+ sp += sizeof(uintptr);
+ addroot((Obj){sp, f->args, 0});
+ }
+ return;
}
}
-
- n = 0;
- while(stk) {
- if(sp < guard-StackGuard || (byte*)stk < sp) {
- runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
- runtime·throw("scanstack");
+ if (ScanStackByFrames) {
+ doframe = false;
+ runtime·gentraceback(pc, sp, nil, gp, 0, nil, 0x7fffffff, addframeroots, &doframe);
+ } else {
+ USED(pc);
+ n = 0;
+ while(stk) {
+ if(sp < guard-StackGuard || (byte*)stk < sp) {
+ runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
+ runtime·throw("scanstack");
+ }
+ addroot((Obj){sp, (byte*)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
+ sp = (byte*)stk->gobuf.sp;
+ guard = stk->stackguard;
+ stk = (Stktop*)stk->stackbase;
+ n++;
}
- addroot((Obj){sp, (byte*)stk - sp, 0});
- sp = (byte*)stk->gobuf.sp;
- guard = stk->stackguard;
- stk = (Stktop*)stk->stackbase;
- n++;
}
}
@@ -1381,14 +1512,17 @@ addroots(void)
for(spanidx=0; spanidx<runtime·mheap->nspan; spanidx++) {
s = allspans[spanidx];
if(s->state == MSpanInUse) {
+ // The garbage collector ignores type pointers stored in MSpan.types:
+ // - Compiler-generated types are stored outside of heap.
+ // - The reflect package has runtime-generated types cached in its data structures.
+ // The garbage collector relies on finding the references via that cache.
switch(s->types.compression) {
case MTypes_Empty:
case MTypes_Single:
break;
case MTypes_Words:
case MTypes_Bytes:
- // TODO(atom): consider using defaultProg instead of 0
- addroot((Obj){(byte*)&s->types.data, sizeof(void*), 0});
+ markonly((byte*)s->types.data);
break;
}
}
@@ -1543,7 +1677,7 @@ sweepspan(ParFor *desc, uint32 idx)
if(cl == 0) {
// Free large span.
runtime·unmarkspan(p, 1<<PageShift);
- *(uintptr*)p = 1; // needs zeroing
+ *(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll; // needs zeroing
runtime·MHeap_Free(runtime·mheap, s, 1);
c->local_alloc -= size;
c->local_nfree++;
@@ -1558,7 +1692,7 @@ sweepspan(ParFor *desc, uint32 idx)
break;
}
if(size > sizeof(uintptr))
- ((uintptr*)p)[1] = 1; // mark as "needs to be zeroed"
+ ((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll; // mark as "needs to be zeroed"
end->next = (MLink*)p;
end = (MLink*)p;
@@ -1655,6 +1789,8 @@ runtime·memorydump(void)
void
runtime·gchelper(void)
{
+ gchelperstart();
+
// parallel mark for over gc roots
runtime·parfordo(work.markfor);
@@ -1668,6 +1804,7 @@ runtime·gchelper(void)
}
runtime·parfordo(work.sweepfor);
+ bufferList[m->helpgc].busy = 0;
if(runtime·xadd(&work.ndone, +1) == work.nproc-1)
runtime·notewakeup(&work.alldone);
}
@@ -1757,6 +1894,8 @@ runtime·gc(int32 force)
// a problem in the past.
if((((uintptr)&work.empty) & 7) != 0)
runtime·throw("runtime: gc work buffer is misaligned");
+ if((((uintptr)&work.full) & 7) != 0)
+ runtime·throw("runtime: gc work buffer is misaligned");
// The gc is turned off (via enablegc) until
// the bootstrap has completed.
@@ -1857,6 +1996,7 @@ gc(struct gc_args *args)
t1 = runtime·nanotime();
+ gchelperstart();
runtime·parfordo(work.markfor);
scanblock(nil, nil, 0, true);
@@ -1868,6 +2008,7 @@ gc(struct gc_args *args)
t2 = runtime·nanotime();
runtime·parfordo(work.sweepfor);
+ bufferList[m->helpgc].busy = 0;
t3 = runtime·nanotime();
if(work.nproc > 1)
@@ -2009,6 +2150,15 @@ runtime∕debug·setGCPercent(intgo in, intgo out)
}
static void
+gchelperstart(void)
+{
+ if(m->helpgc < 0 || m->helpgc >= MaxGcproc)
+ runtime·throw("gchelperstart: bad m->helpgc");
+ if(runtime·xchg(&bufferList[m->helpgc].busy, 1))
+ runtime·throw("gchelperstart: already busy");
+}
+
+static void
runfinq(void)
{
Finalizer *f;
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-26 02:59:37 +0000