diff options
Diffstat (limited to 'src/pkg/runtime/malloc.goc')
| -rw-r--r-- | src/pkg/runtime/malloc.goc | 939 |
1 files changed, 0 insertions, 939 deletions
diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc deleted file mode 100644 index 7b7e350d8..000000000 --- a/src/pkg/runtime/malloc.goc +++ /dev/null @@ -1,939 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// See malloc.h for overview. -// -// TODO(rsc): double-check stats. - -package runtime -#include "runtime.h" -#include "arch_GOARCH.h" -#include "malloc.h" -#include "type.h" -#include "typekind.h" -#include "race.h" -#include "stack.h" -#include "../../cmd/ld/textflag.h" - -// Mark mheap as 'no pointers', it does not contain interesting pointers but occupies ~45K. -#pragma dataflag NOPTR -MHeap runtime·mheap; -#pragma dataflag NOPTR -MStats mstats; - -int32 runtime·checking; - -extern MStats mstats; // defined in zruntime_def_$GOOS_$GOARCH.go - -extern volatile intgo runtime·MemProfileRate; - -static MSpan* largealloc(uint32, uintptr*); -static void profilealloc(void *v, uintptr size); -static void settype(MSpan *s, void *v, uintptr typ); - -// Allocate an object of at least size bytes. -// Small objects are allocated from the per-thread cache's free lists. -// Large objects (> 32 kB) are allocated straight from the heap. -// If the block will be freed with runtime·free(), typ must be 0. -void* -runtime·mallocgc(uintptr size, uintptr typ, uint32 flag) -{ - int32 sizeclass; - uintptr tinysize, size1; - intgo rate; - MCache *c; - MSpan *s; - MLink *v, *next; - byte *tiny; - - if(size == 0) { - // All 0-length allocations use this pointer. - // The language does not require the allocations to - // have distinct values. - return &runtime·zerobase; - } - if(m->mallocing) - runtime·throw("malloc/free - deadlock"); - // Disable preemption during settype. - // We can not use m->mallocing for this, because settype calls mallocgc. - m->locks++; - m->mallocing = 1; - - if(DebugTypeAtBlockEnd) - size += sizeof(uintptr); - - c = m->mcache; - if(!runtime·debug.efence && size <= MaxSmallSize) { - if((flag&(FlagNoScan|FlagNoGC)) == FlagNoScan && size < TinySize) { - // Tiny allocator. - // - // Tiny allocator combines several tiny allocation requests - // into a single memory block. The resulting memory block - // is freed when all subobjects are unreachable. The subobjects - // must be FlagNoScan (don't have pointers), this ensures that - // the amount of potentially wasted memory is bounded. - // - // Size of the memory block used for combining (TinySize) is tunable. - // Current setting is 16 bytes, which relates to 2x worst case memory - // wastage (when all but one subobjects are unreachable). - // 8 bytes would result in no wastage at all, but provides less - // opportunities for combining. - // 32 bytes provides more opportunities for combining, - // but can lead to 4x worst case wastage. - // The best case winning is 8x regardless of block size. - // - // Objects obtained from tiny allocator must not be freed explicitly. - // So when an object will be freed explicitly, we ensure that - // its size >= TinySize. - // - // SetFinalizer has a special case for objects potentially coming - // from tiny allocator, it such case it allows to set finalizers - // for an inner byte of a memory block. - // - // The main targets of tiny allocator are small strings and - // standalone escaping variables. On a json benchmark - // the allocator reduces number of allocations by ~12% and - // reduces heap size by ~20%. - - tinysize = c->tinysize; - if(size <= tinysize) { - tiny = c->tiny; - // Align tiny pointer for required (conservative) alignment. - if((size&7) == 0) - tiny = (byte*)ROUND((uintptr)tiny, 8); - else if((size&3) == 0) - tiny = (byte*)ROUND((uintptr)tiny, 4); - else if((size&1) == 0) - tiny = (byte*)ROUND((uintptr)tiny, 2); - size1 = size + (tiny - c->tiny); - if(size1 <= tinysize) { - // The object fits into existing tiny block. - v = (MLink*)tiny; - c->tiny += size1; - c->tinysize -= size1; - m->mallocing = 0; - m->locks--; - if(m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; - return v; - } - } - // Allocate a new TinySize block. - s = c->alloc[TinySizeClass]; - if(s->freelist == nil) - s = runtime·MCache_Refill(c, TinySizeClass); - v = s->freelist; - next = v->next; - s->freelist = next; - s->ref++; - if(next != nil) // prefetching nil leads to a DTLB miss - PREFETCH(next); - ((uint64*)v)[0] = 0; - ((uint64*)v)[1] = 0; - // See if we need to replace the existing tiny block with the new one - // based on amount of remaining free space. - if(TinySize-size > tinysize) { - c->tiny = (byte*)v + size; - c->tinysize = TinySize - size; - } - size = TinySize; - goto done; - } - // Allocate from mcache free lists. - // Inlined version of SizeToClass(). - if(size <= 1024-8) - sizeclass = runtime·size_to_class8[(size+7)>>3]; - else - sizeclass = runtime·size_to_class128[(size-1024+127) >> 7]; - size = runtime·class_to_size[sizeclass]; - s = c->alloc[sizeclass]; - if(s->freelist == nil) - s = runtime·MCache_Refill(c, sizeclass); - v = s->freelist; - next = v->next; - s->freelist = next; - s->ref++; - if(next != nil) // prefetching nil leads to a DTLB miss - PREFETCH(next); - if(!(flag & FlagNoZero)) { - v->next = nil; - // block is zeroed iff second word is zero ... - if(size > 2*sizeof(uintptr) && ((uintptr*)v)[1] != 0) - runtime·memclr((byte*)v, size); - } - done: - c->local_cachealloc += size; - } else { - // Allocate directly from heap. - s = largealloc(flag, &size); - v = (void*)(s->start << PageShift); - } - - if(flag & FlagNoGC) - runtime·marknogc(v); - else if(!(flag & FlagNoScan)) - runtime·markscan(v); - - if(DebugTypeAtBlockEnd) - *(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ; - - m->mallocing = 0; - // TODO: save type even if FlagNoScan? Potentially expensive but might help - // heap profiling/tracing. - if(UseSpanType && !(flag & FlagNoScan) && typ != 0) - settype(s, v, typ); - - if(raceenabled) - runtime·racemalloc(v, size); - - if(runtime·debug.allocfreetrace) - runtime·tracealloc(v, size, typ); - - if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) { - if(size < rate && size < c->next_sample) - c->next_sample -= size; - else - profilealloc(v, size); - } - - m->locks--; - if(m->locks == 0 && g->preempt) // restore the preemption request in case we've cleared it in newstack - g->stackguard0 = StackPreempt; - - if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc) - runtime·gc(0); - - return v; -} - -static MSpan* -largealloc(uint32 flag, uintptr *sizep) -{ - uintptr npages, size; - MSpan *s; - void *v; - - // Allocate directly from heap. - size = *sizep; - if(size + PageSize < size) - runtime·throw("out of memory"); - npages = size >> PageShift; - if((size & PageMask) != 0) - npages++; - s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZero)); - if(s == nil) - runtime·throw("out of memory"); - s->limit = (byte*)(s->start<<PageShift) + size; - *sizep = npages<<PageShift; - v = (void*)(s->start << PageShift); - // setup for mark sweep - runtime·markspan(v, 0, 0, true); - return s; -} - -static void -profilealloc(void *v, uintptr size) -{ - uintptr rate; - int32 next; - MCache *c; - - c = m->mcache; - rate = runtime·MemProfileRate; - if(size < rate) { - // pick next profile time - // If you change this, also change allocmcache. - if(rate > 0x3fffffff) // make 2*rate not overflow - rate = 0x3fffffff; - next = runtime·fastrand1() % (2*rate); - // Subtract the "remainder" of the current allocation. - // Otherwise objects that are close in size to sampling rate - // will be under-sampled, because we consistently discard this remainder. - next -= (size - c->next_sample); - if(next < 0) - next = 0; - c->next_sample = next; - } - runtime·MProf_Malloc(v, size); -} - -void* -runtime·malloc(uintptr size) -{ - return runtime·mallocgc(size, 0, FlagNoInvokeGC); -} - -// Free the object whose base pointer is v. -void -runtime·free(void *v) -{ - int32 sizeclass; - MSpan *s; - MCache *c; - uintptr size; - - if(v == nil) - return; - - // If you change this also change mgc0.c:/^sweep, - // which has a copy of the guts of free. - - if(m->mallocing) - runtime·throw("malloc/free - deadlock"); - m->mallocing = 1; - - if(!runtime·mlookup(v, nil, nil, &s)) { - runtime·printf("free %p: not an allocated block\n", v); - runtime·throw("free runtime·mlookup"); - } - size = s->elemsize; - sizeclass = s->sizeclass; - // Objects that are smaller than TinySize can be allocated using tiny alloc, - // if then such object is combined with an object with finalizer, we will crash. - if(size < TinySize) - runtime·throw("freeing too small block"); - - if(runtime·debug.allocfreetrace) - runtime·tracefree(v, size); - - // Ensure that the span is swept. - // If we free into an unswept span, we will corrupt GC bitmaps. - runtime·MSpan_EnsureSwept(s); - - if(s->specials != nil) - runtime·freeallspecials(s, v, size); - - c = m->mcache; - if(sizeclass == 0) { - // Large object. - s->needzero = 1; - // Must mark v freed before calling unmarkspan and MHeap_Free: - // they might coalesce v into other spans and change the bitmap further. - runtime·markfreed(v); - runtime·unmarkspan(v, 1<<PageShift); - // NOTE(rsc,dvyukov): The original implementation of efence - // in CL 22060046 used SysFree instead of SysFault, so that - // the operating system would eventually give the memory - // back to us again, so that an efence program could run - // longer without running out of memory. Unfortunately, - // calling SysFree here without any kind of adjustment of the - // heap data structures means that when the memory does - // come back to us, we have the wrong metadata for it, either in - // the MSpan structures or in the garbage collection bitmap. - // Using SysFault here means that the program will run out of - // memory fairly quickly in efence mode, but at least it won't - // have mysterious crashes due to confused memory reuse. - // It should be possible to switch back to SysFree if we also - // implement and then call some kind of MHeap_DeleteSpan. - if(runtime·debug.efence) - runtime·SysFault((void*)(s->start<<PageShift), size); - else - runtime·MHeap_Free(&runtime·mheap, s, 1); - c->local_nlargefree++; - c->local_largefree += size; - } else { - // Small object. - if(size > 2*sizeof(uintptr)) - ((uintptr*)v)[1] = (uintptr)0xfeedfeedfeedfeedll; // mark as "needs to be zeroed" - else if(size > sizeof(uintptr)) - ((uintptr*)v)[1] = 0; - // Must mark v freed before calling MCache_Free: - // it might coalesce v and other blocks into a bigger span - // and change the bitmap further. - c->local_nsmallfree[sizeclass]++; - c->local_cachealloc -= size; - if(c->alloc[sizeclass] == s) { - // We own the span, so we can just add v to the freelist - runtime·markfreed(v); - ((MLink*)v)->next = s->freelist; - s->freelist = v; - s->ref--; - } else { - // Someone else owns this span. Add to free queue. - runtime·MCache_Free(c, v, sizeclass, size); - } - } - m->mallocing = 0; -} - -int32 -runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) -{ - uintptr n, i; - byte *p; - MSpan *s; - - m->mcache->local_nlookup++; - if (sizeof(void*) == 4 && m->mcache->local_nlookup >= (1<<30)) { - // purge cache stats to prevent overflow - runtime·lock(&runtime·mheap); - runtime·purgecachedstats(m->mcache); - runtime·unlock(&runtime·mheap); - } - - s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); - if(sp) - *sp = s; - if(s == nil) { - runtime·checkfreed(v, 1); - if(base) - *base = nil; - if(size) - *size = 0; - return 0; - } - - p = (byte*)((uintptr)s->start<<PageShift); - if(s->sizeclass == 0) { - // Large object. - if(base) - *base = p; - if(size) - *size = s->npages<<PageShift; - return 1; - } - - n = s->elemsize; - if(base) { - i = ((byte*)v - p)/n; - *base = p + i*n; - } - if(size) - *size = n; - - return 1; -} - -void -runtime·purgecachedstats(MCache *c) -{ - MHeap *h; - int32 i; - - // Protected by either heap or GC lock. - h = &runtime·mheap; - mstats.heap_alloc += c->local_cachealloc; - c->local_cachealloc = 0; - mstats.nlookup += c->local_nlookup; - c->local_nlookup = 0; - h->largefree += c->local_largefree; - c->local_largefree = 0; - h->nlargefree += c->local_nlargefree; - c->local_nlargefree = 0; - for(i=0; i<nelem(c->local_nsmallfree); i++) { - h->nsmallfree[i] += c->local_nsmallfree[i]; - c->local_nsmallfree[i] = 0; - } -} - -// Size of the trailing by_size array differs between Go and C, -// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility. -// sizeof_C_MStats is what C thinks about size of Go struct. -uintptr runtime·sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeof(mstats.by_size[0]); - -#define MaxArena32 (2U<<30) - -void -runtime·mallocinit(void) -{ - byte *p, *p1; - uintptr arena_size, bitmap_size, spans_size, p_size; - extern byte end[]; - uintptr limit; - uint64 i; - bool reserved; - - p = nil; - p_size = 0; - arena_size = 0; - bitmap_size = 0; - spans_size = 0; - reserved = false; - - // for 64-bit build - USED(p); - USED(p_size); - USED(arena_size); - USED(bitmap_size); - USED(spans_size); - - runtime·InitSizes(); - - if(runtime·class_to_size[TinySizeClass] != TinySize) - runtime·throw("bad TinySizeClass"); - - // limit = runtime·memlimit(); - // See https://code.google.com/p/go/issues/detail?id=5049 - // TODO(rsc): Fix after 1.1. - limit = 0; - - // Set up the allocation arena, a contiguous area of memory where - // allocated data will be found. The arena begins with a bitmap large - // enough to hold 4 bits per allocated word. - if(sizeof(void*) == 8 && (limit == 0 || limit > (1<<30))) { - // On a 64-bit machine, allocate from a single contiguous reservation. - // 128 GB (MaxMem) should be big enough for now. - // - // The code will work with the reservation at any address, but ask - // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). - // Allocating a 128 GB region takes away 37 bits, and the amd64 - // doesn't let us choose the top 17 bits, so that leaves the 11 bits - // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means - // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. - // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid - // UTF-8 sequences, and they are otherwise as far away from - // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 - // addresses. An earlier attempt to use 0x11f8 caused out of memory errors - // on OS X during thread allocations. 0x00c0 causes conflicts with - // AddressSanitizer which reserves all memory up to 0x0100. - // These choices are both for debuggability and to reduce the - // odds of the conservative garbage collector not collecting memory - // because some non-pointer block of memory had a bit pattern - // that matched a memory address. - // - // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) - // but it hardly matters: e0 00 is not valid UTF-8 either. - // - // If this fails we fall back to the 32 bit memory mechanism - arena_size = MaxMem; - bitmap_size = arena_size / (sizeof(void*)*8/4); - spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[0]); - spans_size = ROUND(spans_size, PageSize); - for(i = 0; i <= 0x7f; i++) { - p = (void*)(i<<40 | 0x00c0ULL<<32); - p_size = bitmap_size + spans_size + arena_size + PageSize; - p = runtime·SysReserve(p, p_size, &reserved); - if(p != nil) - break; - } - } - if (p == nil) { - // On a 32-bit machine, we can't typically get away - // with a giant virtual address space reservation. - // Instead we map the memory information bitmap - // immediately after the data segment, large enough - // to handle another 2GB of mappings (256 MB), - // along with a reservation for another 512 MB of memory. - // When that gets used up, we'll start asking the kernel - // for any memory anywhere and hope it's in the 2GB - // following the bitmap (presumably the executable begins - // near the bottom of memory, so we'll have to use up - // most of memory before the kernel resorts to giving out - // memory before the beginning of the text segment). - // - // Alternatively we could reserve 512 MB bitmap, enough - // for 4GB of mappings, and then accept any memory the - // kernel threw at us, but normally that's a waste of 512 MB - // of address space, which is probably too much in a 32-bit world. - bitmap_size = MaxArena32 / (sizeof(void*)*8/4); - arena_size = 512<<20; - spans_size = MaxArena32 / PageSize * sizeof(runtime·mheap.spans[0]); - if(limit > 0 && arena_size+bitmap_size+spans_size > limit) { - bitmap_size = (limit / 9) & ~((1<<PageShift) - 1); - arena_size = bitmap_size * 8; - spans_size = arena_size / PageSize * sizeof(runtime·mheap.spans[0]); - } - spans_size = ROUND(spans_size, PageSize); - - // SysReserve treats the address we ask for, end, as a hint, - // not as an absolute requirement. If we ask for the end - // of the data segment but the operating system requires - // a little more space before we can start allocating, it will - // give out a slightly higher pointer. Except QEMU, which - // is buggy, as usual: it won't adjust the pointer upward. - // So adjust it upward a little bit ourselves: 1/4 MB to get - // away from the running binary image and then round up - // to a MB boundary. - p = (byte*)ROUND((uintptr)end + (1<<18), 1<<20); - p_size = bitmap_size + spans_size + arena_size + PageSize; - p = runtime·SysReserve(p, p_size, &reserved); - if(p == nil) - runtime·throw("runtime: cannot reserve arena virtual address space"); - } - - // PageSize can be larger than OS definition of page size, - // so SysReserve can give us a PageSize-unaligned pointer. - // To overcome this we ask for PageSize more and round up the pointer. - p1 = (byte*)ROUND((uintptr)p, PageSize); - - runtime·mheap.spans = (MSpan**)p1; - runtime·mheap.bitmap = p1 + spans_size; - runtime·mheap.arena_start = p1 + spans_size + bitmap_size; - runtime·mheap.arena_used = runtime·mheap.arena_start; - runtime·mheap.arena_end = p + p_size; - runtime·mheap.arena_reserved = reserved; - - if(((uintptr)runtime·mheap.arena_start & (PageSize-1)) != 0) - runtime·throw("misrounded allocation in mallocinit"); - - // Initialize the rest of the allocator. - runtime·MHeap_Init(&runtime·mheap); - m->mcache = runtime·allocmcache(); - - // See if it works. - runtime·free(runtime·malloc(TinySize)); -} - -void* -runtime·MHeap_SysAlloc(MHeap *h, uintptr n) -{ - byte *p, *p_end; - uintptr p_size; - bool reserved; - - if(n > h->arena_end - h->arena_used) { - // We are in 32-bit mode, maybe we didn't use all possible address space yet. - // Reserve some more space. - byte *new_end; - - p_size = ROUND(n + PageSize, 256<<20); - new_end = h->arena_end + p_size; - if(new_end <= h->arena_start + MaxArena32) { - // TODO: It would be bad if part of the arena - // is reserved and part is not. - p = runtime·SysReserve(h->arena_end, p_size, &reserved); - if(p == h->arena_end) { - h->arena_end = new_end; - h->arena_reserved = reserved; - } - else if(p+p_size <= h->arena_start + MaxArena32) { - // Keep everything page-aligned. - // Our pages are bigger than hardware pages. - h->arena_end = p+p_size; - h->arena_used = p + (-(uintptr)p&(PageSize-1)); - h->arena_reserved = reserved; - } else { - uint64 stat; - stat = 0; - runtime·SysFree(p, p_size, &stat); - } - } - } - if(n <= h->arena_end - h->arena_used) { - // Keep taking from our reservation. - p = h->arena_used; - runtime·SysMap(p, n, h->arena_reserved, &mstats.heap_sys); - h->arena_used += n; - runtime·MHeap_MapBits(h); - runtime·MHeap_MapSpans(h); - if(raceenabled) - runtime·racemapshadow(p, n); - - if(((uintptr)p & (PageSize-1)) != 0) - runtime·throw("misrounded allocation in MHeap_SysAlloc"); - return p; - } - - // If using 64-bit, our reservation is all we have. - if(h->arena_end - h->arena_start >= MaxArena32) - return nil; - - // On 32-bit, once the reservation is gone we can - // try to get memory at a location chosen by the OS - // and hope that it is in the range we allocated bitmap for. - p_size = ROUND(n, PageSize) + PageSize; - p = runtime·SysAlloc(p_size, &mstats.heap_sys); - if(p == nil) - return nil; - - if(p < h->arena_start || p+p_size - h->arena_start >= MaxArena32) { - runtime·printf("runtime: memory allocated by OS (%p) not in usable range [%p,%p)\n", - p, h->arena_start, h->arena_start+MaxArena32); - runtime·SysFree(p, p_size, &mstats.heap_sys); - return nil; - } - - p_end = p + p_size; - p += -(uintptr)p & (PageSize-1); - if(p+n > h->arena_used) { - h->arena_used = p+n; - if(p_end > h->arena_end) - h->arena_end = p_end; - runtime·MHeap_MapBits(h); - runtime·MHeap_MapSpans(h); - if(raceenabled) - runtime·racemapshadow(p, n); - } - - if(((uintptr)p & (PageSize-1)) != 0) - runtime·throw("misrounded allocation in MHeap_SysAlloc"); - return p; -} - -static struct -{ - Lock; - byte* pos; - byte* end; -} persistent; - -enum -{ - PersistentAllocChunk = 256<<10, - PersistentAllocMaxBlock = 64<<10, // VM reservation granularity is 64K on windows -}; - -// Wrapper around SysAlloc that can allocate small chunks. -// There is no associated free operation. -// Intended for things like function/type/debug-related persistent data. -// If align is 0, uses default align (currently 8). -void* -runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat) -{ - byte *p; - - if(align != 0) { - if(align&(align-1)) - runtime·throw("persistentalloc: align is not a power of 2"); - if(align > PageSize) - runtime·throw("persistentalloc: align is too large"); - } else - align = 8; - if(size >= PersistentAllocMaxBlock) - return runtime·SysAlloc(size, stat); - runtime·lock(&persistent); - persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align); - if(persistent.pos + size > persistent.end) { - persistent.pos = runtime·SysAlloc(PersistentAllocChunk, &mstats.other_sys); - if(persistent.pos == nil) { - runtime·unlock(&persistent); - runtime·throw("runtime: cannot allocate memory"); - } - persistent.end = persistent.pos + PersistentAllocChunk; - } - p = persistent.pos; - persistent.pos += size; - runtime·unlock(&persistent); - if(stat != &mstats.other_sys) { - // reaccount the allocation against provided stat - runtime·xadd64(stat, size); - runtime·xadd64(&mstats.other_sys, -(uint64)size); - } - return p; -} - -static void -settype(MSpan *s, void *v, uintptr typ) -{ - uintptr size, ofs, j, t; - uintptr ntypes, nbytes2, nbytes3; - uintptr *data2; - byte *data3; - - if(s->sizeclass == 0) { - s->types.compression = MTypes_Single; - s->types.data = typ; - return; - } - size = s->elemsize; - ofs = ((uintptr)v - (s->start<<PageShift)) / size; - - switch(s->types.compression) { - case MTypes_Empty: - ntypes = (s->npages << PageShift) / size; - nbytes3 = 8*sizeof(uintptr) + 1*ntypes; - data3 = runtime·mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC); - s->types.compression = MTypes_Bytes; - s->types.data = (uintptr)data3; - ((uintptr*)data3)[1] = typ; - data3[8*sizeof(uintptr) + ofs] = 1; - break; - - case MTypes_Words: - ((uintptr*)s->types.data)[ofs] = typ; - break; - - case MTypes_Bytes: - data3 = (byte*)s->types.data; - for(j=1; j<8; j++) { - if(((uintptr*)data3)[j] == typ) { - break; - } - if(((uintptr*)data3)[j] == 0) { - ((uintptr*)data3)[j] = typ; - break; - } - } - if(j < 8) { - data3[8*sizeof(uintptr) + ofs] = j; - } else { - ntypes = (s->npages << PageShift) / size; - nbytes2 = ntypes * sizeof(uintptr); - data2 = runtime·mallocgc(nbytes2, 0, FlagNoProfiling|FlagNoScan|FlagNoInvokeGC); - s->types.compression = MTypes_Words; - s->types.data = (uintptr)data2; - - // Move the contents of data3 to data2. Then deallocate data3. - for(j=0; j<ntypes; j++) { - t = data3[8*sizeof(uintptr) + j]; - t = ((uintptr*)data3)[t]; - data2[j] = t; - } - data2[ofs] = typ; - } - break; - } -} - -uintptr -runtime·gettype(void *v) -{ - MSpan *s; - uintptr t, ofs; - byte *data; - - s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); - if(s != nil) { - t = 0; - switch(s->types.compression) { - case MTypes_Empty: - break; - case MTypes_Single: - t = s->types.data; - break; - case MTypes_Words: - ofs = (uintptr)v - (s->start<<PageShift); - t = ((uintptr*)s->types.data)[ofs/s->elemsize]; - break; - case MTypes_Bytes: - ofs = (uintptr)v - (s->start<<PageShift); - data = (byte*)s->types.data; - t = data[8*sizeof(uintptr) + ofs/s->elemsize]; - t = ((uintptr*)data)[t]; - break; - default: - runtime·throw("runtime·gettype: invalid compression kind"); - } - if(0) { - runtime·printf("%p -> %d,%X\n", v, (int32)s->types.compression, (int64)t); - } - return t; - } - return 0; -} - -// Runtime stubs. - -void* -runtime·mal(uintptr n) -{ - return runtime·mallocgc(n, 0, 0); -} - -#pragma textflag NOSPLIT -func new(typ *Type) (ret *uint8) { - ret = runtime·mallocgc(typ->size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&KindNoPointers ? FlagNoScan : 0); -} - -static void* -cnew(Type *typ, intgo n, int32 objtyp) -{ - if((objtyp&(PtrSize-1)) != objtyp) - runtime·throw("runtime: invalid objtyp"); - if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size)) - runtime·panicstring("runtime: allocation size out of range"); - return runtime·mallocgc(typ->size*n, (uintptr)typ | objtyp, typ->kind&KindNoPointers ? FlagNoScan : 0); -} - -// same as runtime·new, but callable from C -void* -runtime·cnew(Type *typ) -{ - return cnew(typ, 1, TypeInfo_SingleObject); -} - -void* -runtime·cnewarray(Type *typ, intgo n) -{ - return cnew(typ, n, TypeInfo_Array); -} - -func GC() { - runtime·gc(2); // force GC and do eager sweep -} - -func SetFinalizer(obj Eface, finalizer Eface) { - byte *base; - uintptr size; - FuncType *ft; - int32 i; - uintptr nret; - Type *t; - Type *fint; - PtrType *ot; - Iface iface; - - if(obj.type == nil) { - runtime·printf("runtime.SetFinalizer: first argument is nil interface\n"); - goto throw; - } - if(obj.type->kind != KindPtr) { - runtime·printf("runtime.SetFinalizer: first argument is %S, not pointer\n", *obj.type->string); - goto throw; - } - ot = (PtrType*)obj.type; - // As an implementation detail we do not run finalizers for zero-sized objects, - // because we use &runtime·zerobase for all such allocations. - if(ot->elem != nil && ot->elem->size == 0) - return; - // The following check is required for cases when a user passes a pointer to composite literal, - // but compiler makes it a pointer to global. For example: - // var Foo = &Object{} - // func main() { - // runtime.SetFinalizer(Foo, nil) - // } - // See issue 7656. - if((byte*)obj.data < runtime·mheap.arena_start || runtime·mheap.arena_used <= (byte*)obj.data) - return; - if(!runtime·mlookup(obj.data, &base, &size, nil) || obj.data != base) { - // As an implementation detail we allow to set finalizers for an inner byte - // of an object if it could come from tiny alloc (see mallocgc for details). - if(ot->elem == nil || (ot->elem->kind&KindNoPointers) == 0 || ot->elem->size >= TinySize) { - runtime·printf("runtime.SetFinalizer: pointer not at beginning of allocated block (%p)\n", obj.data); - goto throw; - } - } - if(finalizer.type != nil) { - runtime·createfing(); - if(finalizer.type->kind != KindFunc) - goto badfunc; - ft = (FuncType*)finalizer.type; - if(ft->dotdotdot || ft->in.len != 1) - goto badfunc; - fint = *(Type**)ft->in.array; - if(fint == obj.type) { - // ok - same type - } else if(fint->kind == KindPtr && (fint->x == nil || fint->x->name == nil || obj.type->x == nil || obj.type->x->name == nil) && ((PtrType*)fint)->elem == ((PtrType*)obj.type)->elem) { - // ok - not same type, but both pointers, - // one or the other is unnamed, and same element type, so assignable. - } else if(fint->kind == KindInterface && ((InterfaceType*)fint)->mhdr.len == 0) { - // ok - satisfies empty interface - } else if(fint->kind == KindInterface && runtime·ifaceE2I2((InterfaceType*)fint, obj, &iface)) { - // ok - satisfies non-empty interface - } else - goto badfunc; - - // compute size needed for return parameters - nret = 0; - for(i=0; i<ft->out.len; i++) { - t = ((Type**)ft->out.array)[i]; - nret = ROUND(nret, t->align) + t->size; - } - nret = ROUND(nret, sizeof(void*)); - ot = (PtrType*)obj.type; - if(!runtime·addfinalizer(obj.data, finalizer.data, nret, fint, ot)) { - runtime·printf("runtime.SetFinalizer: finalizer already set\n"); - goto throw; - } - } else { - // NOTE: asking to remove a finalizer when there currently isn't one set is OK. - runtime·removefinalizer(obj.data); - } - return; - -badfunc: - runtime·printf("runtime.SetFinalizer: cannot pass %S to finalizer %S\n", *obj.type->string, *finalizer.type->string); -throw: - runtime·throw("runtime.SetFinalizer"); -} |