1 files changed, 88 insertions, 1 deletions

diff --git a/src/pkg/runtime/mheap.c b/src/pkg/runtime/mheap.c
index 37d505681..c877bfca9 100644
--- a/src/pkg/runtime/mheap.c
+++ b/src/pkg/runtime/mheap.c
@@ -13,6 +13,7 @@
 // and heapmap(i) == span for all s->start <= i < s->start+s->npages.
 
 #include "runtime.h"
+#include "arch_GOARCH.h"
 #include "malloc.h"
 
 static MSpan *MHeap_AllocLocked(MHeap*, uintptr, int32);
@@ -101,6 +102,9 @@ HaveSpan:
 		runtime·throw("MHeap_AllocLocked - bad npages");
 	runtime·MSpanList_Remove(s);
 	s->state = MSpanInUse;
+	mstats.heap_idle -= s->npages<<PageShift;
+	mstats.heap_released -= s->npreleased<<PageShift;
+	s->npreleased = 0;
 
 	if(s->npages > npage) {
 		// Trim extra and put it back in the heap.
@@ -276,7 +280,10 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
 		runtime·printf("MHeap_FreeLocked - span %p ptr %p state %d ref %d\n", s, s->start<<PageShift, s->state, s->ref);
 		runtime·throw("MHeap_FreeLocked - invalid free");
 	}
+	mstats.heap_idle += s->npages<<PageShift;
 	s->state = MSpanFree;
+	s->unusedsince = 0;
+	s->npreleased = 0;
 	runtime·MSpanList_Remove(s);
 	sp = (uintptr*)(s->start<<PageShift);
 
@@ -289,6 +296,7 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
 		*tp |= *sp;	// propagate "needs zeroing" mark
 		s->start = t->start;
 		s->npages += t->npages;
+		s->npreleased = t->npreleased; // absorb released pages
 		p -= t->npages;
 		h->map[p] = s;
 		runtime·MSpanList_Remove(t);
@@ -301,6 +309,7 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
 		tp = (uintptr*)(t->start<<PageShift);
 		*sp |= *tp;	// propagate "needs zeroing" mark
 		s->npages += t->npages;
+		s->npreleased += t->npreleased;
 		h->map[p + s->npages - 1] = s;
 		runtime·MSpanList_Remove(t);
 		t->state = MSpanDead;
@@ -314,8 +323,84 @@ MHeap_FreeLocked(MHeap *h, MSpan *s)
 		runtime·MSpanList_Insert(&h->free[s->npages], s);
 	else
 		runtime·MSpanList_Insert(&h->large, s);
+}
 
-	// TODO(rsc): IncrementalScavenge() to return memory to OS.
+// Release (part of) unused memory to OS.
+// Goroutine created at startup.
+// Loop forever.
+void
+runtime·MHeap_Scavenger(void)
+{
+	MHeap *h;
+	MSpan *s, *list;
+	uint64 tick, now, forcegc, limit;
+	uint32 k, i;
+	uintptr released, sumreleased;
+	byte *env;
+	bool trace;
+	Note note;
+
+	// If we go two minutes without a garbage collection, force one to run.
+	forcegc = 2*60*1e9;
+	// If a span goes unused for 5 minutes after a garbage collection,
+	// we hand it back to the operating system.
+	limit = 5*60*1e9;
+	// Make wake-up period small enough for the sampling to be correct.
+	if(forcegc < limit)
+		tick = forcegc/2;
+	else
+		tick = limit/2;
+
+	trace = false;
+	env = runtime·getenv("GOGCTRACE");
+	if(env != nil)
+		trace = runtime·atoi(env) > 0;
+
+	h = &runtime·mheap;
+	for(k=0;; k++) {
+		runtime·noteclear(&note);
+		runtime·entersyscall();
+		runtime·notetsleep(&note, tick);
+		runtime·exitsyscall();
+
+		runtime·lock(h);
+		now = runtime·nanotime();
+		if(now - mstats.last_gc > forcegc) {
+			runtime·unlock(h);
+			runtime·gc(1);
+			runtime·lock(h);
+			now = runtime·nanotime();
+			if (trace)
+				runtime·printf("scvg%d: GC forced\n", k);
+		}
+		sumreleased = 0;
+		for(i=0; i < nelem(h->free)+1; i++) {
+			if(i < nelem(h->free))
+				list = &h->free[i];
+			else
+				list = &h->large;
+			if(runtime·MSpanList_IsEmpty(list))
+				continue;
+			for(s=list->next; s != list; s=s->next) {
+				if(s->unusedsince != 0 && (now - s->unusedsince) > limit) {
+					released = (s->npages - s->npreleased) << PageShift;
+					mstats.heap_released += released;
+					sumreleased += released;
+					s->npreleased = s->npages;
+					runtime·SysUnused((void*)(s->start << PageShift), s->npages << PageShift);
+				}
+			}
+		}
+		runtime·unlock(h);
+
+		if(trace) {
+			if(sumreleased > 0)
+				runtime·printf("scvg%d: %p MB released\n", k, sumreleased>>20);
+			runtime·printf("scvg%d: inuse: %D, idle: %D, sys: %D, released: %D, consumed: %D (MB)\n",
+				k, mstats.heap_inuse>>20, mstats.heap_idle>>20, mstats.heap_sys>>20,
+				mstats.heap_released>>20, (mstats.heap_sys - mstats.heap_released)>>20);
+		}
+	}
 }
 
 // Initialize a new span with the given start and npages.
@@ -330,6 +415,8 @@ runtime·MSpan_Init(MSpan *span, PageID start, uintptr npages)
 	span->ref = 0;
 	span->sizeclass = 0;
 	span->state = 0;
+	span->unusedsince = 0;
+	span->npreleased = 0;
 }
 
 // Initialize an empty doubly-linked list.