void FixedAlloc::Destroy()
{
// Free all of the blocks
while (m_firstBlock) {
#ifdef MMGC_MEMORY_PROFILER
if(m_firstBlock->numAlloc > 0 && m_heap->GetStatus() != kMemAbort) {
union {
char* mem_c;
uint32_t* mem;
};
mem_c = m_firstBlock->items;
unsigned int itemNum = 0;
while(itemNum++ < m_itemsPerBlock) {
if(IsInUse(m_firstBlock, mem)) {
GCLog("Leaked %d byte item. Addr: 0x%p\n", GetItemSize(), GetUserPointer(mem));
PrintAllocStackTrace(GetUserPointer(mem));
}
mem_c += m_itemSize;
}
}
#ifdef MMGC_MEMORY_INFO
//check for writes on deleted memory
VerifyFreeBlockIntegrity(m_firstBlock->firstFree, m_firstBlock->size);
#endif
#endif
FreeChunk(m_firstBlock);
}
m_firstBlock = NULL;
}
GCAlloc::~GCAlloc()
{
CoalesceQuickList();
// Free all of the blocks
GCAssertMsg(GetNumAlloc() == 0, "You have leaks");
while (m_firstBlock) {
#ifdef MMGC_MEMORY_INFO
//check where any item within this block wasn't written to after being poisoned
VerifyFreeBlockIntegrity(m_firstBlock->firstFree, m_firstBlock->size);
#endif //MMGC_MEMORY_INFO
GCBlock *b = m_firstBlock;
UnlinkChunk(b);
FreeChunk(b);
}
}
void FixedAlloc::Destroy()
{
// Free all of the blocks
while (m_firstBlock) {
#ifdef MMGC_MEMORY_PROFILER
if(m_firstBlock->numAlloc > 0 && m_heap->GetStatus() != kMemAbort) {
union {
char* mem_c;
uint32_t* mem;
};
mem_c = m_firstBlock->items;
unsigned int itemNum = 0;
while(itemNum++ < m_itemsPerBlock) {
if(IsInUse(m_firstBlock, mem)) {
// supress output in release build UNLESS the profiler is on
#ifndef GCDEBUG
if(m_heap->GetProfiler() != NULL)
#endif
{
GCLog("Leaked %d byte item. Addr: 0x%p\n", GetItemSize(), GetUserPointer(mem));
PrintAllocStackTrace(GetUserPointer(mem));
}
}
mem_c += m_itemSize;
}
}
#ifdef MMGC_MEMORY_INFO
//check for writes on deleted memory
VerifyFreeBlockIntegrity(m_firstBlock->firstFree, m_firstBlock->size);
#endif
#endif
// Note, don't cache any state across this call; FreeChunk may temporarily
// release locks held if the true type of this allocator is FixedAllocSafe.
FreeChunk(m_firstBlock);
}
m_firstBlock = NULL;
}
GCAlloc::GCBlock* GCAlloc::CreateChunk(int flags)
{
// Too many definitions of kBlockSize, make sure they're at least in sync.
GCAssert(uint32_t(kBlockSize) == GCHeap::kBlockSize);
// Get bitmap space; this may trigger OOM handling.
gcbits_t* bits = m_bitsInPage ? NULL : (gcbits_t*)m_gc->AllocBits(m_numBitmapBytes, m_sizeClassIndex);
// Allocate a new block; this may trigger OOM handling (though that
// won't affect the bitmap space, which is not GC'd individually).
GCBlock* b = (GCBlock*) m_gc->AllocBlock(1, PageMap::kGCAllocPage, /*zero*/true, (flags&GC::kCanFail) != 0);
if (b)
{
VALGRIND_CREATE_MEMPOOL(b, 0/*redZoneSize*/, 1/*zeroed*/);
// treat block header as a separate allocation
VALGRIND_MEMPOOL_ALLOC(b, b, sizeof(GCBlock));
b->gc = m_gc;
b->alloc = this;
b->size = m_itemSize;
b->slowFlags = 0;
if(m_gc->collecting && m_finalized)
b->finalizeState = m_gc->finalizedValue;
else
b->finalizeState = !m_gc->finalizedValue;
b->bibopTag = m_bibopTag;
#ifdef MMGC_FASTBITS
b->bitsShift = (uint8_t) m_bitsShift;
#endif
b->containsPointers = ContainsPointers();
b->rcobject = ContainsRCObjects();
if (m_bitsInPage)
b->bits = (gcbits_t*)b + sizeof(GCBlock);
else
b->bits = bits;
// ditto for in page bits
if (m_bitsInPage) {
VALGRIND_MEMPOOL_ALLOC(b, b->bits, m_numBitmapBytes);
}
// Link the block at the end of the list
b->prev = m_lastBlock;
b->next = 0;
if (m_lastBlock) {
m_lastBlock->next = b;
}
if (!m_firstBlock) {
m_firstBlock = b;
}
m_lastBlock = b;
// Add our new ChunkBlock to the firstFree list (which should be empty)
if (m_firstFree)
{
GCAssert(m_firstFree->prevFree == 0);
m_firstFree->prevFree = b;
}
b->nextFree = m_firstFree;
b->prevFree = 0;
m_firstFree = b;
// calculate back from end (better alignment, no dead space at end)
b->items = (char*)b+GCHeap::kBlockSize - m_itemsPerBlock * m_itemSize;
b->numFree = (short)m_itemsPerBlock;
// explode the new block onto its free list
//
// We must make the object look free, which means poisoning it properly and setting
// the mark bits correctly.
b->firstFree = b->items;
void** p = (void**)(void*)b->items;
int limit = m_itemsPerBlock-1;
#ifdef MMGC_HOOKS
GCHeap* heap = GCHeap::GetGCHeap();
#endif
for ( int i=0 ; i < limit ; i++ ) {
#ifdef MMGC_HOOKS
#ifdef MMGC_MEMORY_INFO // DebugSize is 0 if MEMORY_INFO is off, so we get an "obviously true" warning from GCC.
GCAssert(m_itemSize >= DebugSize());
#endif
if(heap->HooksEnabled())
heap->PseudoFreeHook(GetUserPointer(p), m_itemSize - DebugSize(), uint8_t(GCHeap::GCSweptPoison));
#endif
p = FLSeed(p, (char*)p + m_itemSize);
}
#ifdef MMGC_HOOKS
if(heap->HooksEnabled())
heap->PseudoFreeHook(GetUserPointer(p), m_itemSize - DebugSize(), uint8_t(GCHeap::GCSweptPoison));
#endif
p[0] = NULL;
// Set all the mark bits to 'free'
GCAssert(sizeof(gcbits_t) == 1);
GCAssert(kFreelist == 3);
GCAssert(m_numBitmapBytes % 4 == 0);
uint32_t *pbits = (uint32_t*)(void *)b->bits;
for(int i=0, n=m_numBitmapBytes>>2; i < n; i++)
pbits[i] = 0x03030303;
#ifdef MMGC_MEMORY_INFO
VerifyFreeBlockIntegrity(b->firstFree, m_itemSize);
#endif
}
else {
if (bits)
void* GCAlloc::Alloc(int flags)
#endif
{
GCAssertMsg(((size_t)m_itemSize >= size), "allocator itemsize too small");
// Allocation must be signalled before we allocate because no GC work must be allowed to
// come between an allocation and an initialization - if it does, we may crash, as
// GCFinalizedObject subclasses may not have a valid vtable, but the GC depends on them
// having it. In principle we could signal allocation late but only set the object
// flags after signaling, but we might still cause trouble for the profiler, which also
// depends on non-interruptibility.
m_gc->SignalAllocWork(m_itemSize);
GCBlock* b = m_firstFree;
start:
if (b == NULL) {
if (m_needsSweeping && !m_gc->collecting) {
Sweep(m_needsSweeping);
b = m_firstFree;
goto start;
}
bool canFail = (flags & GC::kCanFail) != 0;
CreateChunk(canFail);
b = m_firstFree;
if (b == NULL) {
GCAssert(canFail);
return NULL;
}
}
GCAssert(!b->needsSweeping);
GCAssert(b == m_firstFree);
GCAssert(b && !b->IsFull());
void *item;
if(b->firstFree) {
item = b->firstFree;
b->firstFree = *((void**)item);
// clear free list pointer, the rest was zero'd in free
*(intptr_t*) item = 0;
#ifdef MMGC_MEMORY_INFO
//check for writes on deleted memory
VerifyFreeBlockIntegrity(item, b->size);
#endif
} else {
item = b->nextItem;
if(((uintptr_t)((char*)item + b->size) & 0xfff) != 0) {
b->nextItem = (char*)item + b->size;
} else {
b->nextItem = NULL;
}
}
// set up bits, items start out white and whether they need finalization
// is determined by the caller
// make sure we ended up in the right place
GCAssert(((flags&GC::kContainsPointers) != 0) == ContainsPointers());
// this assumes what we assert
GCAssert((unsigned long)GC::kFinalize == (unsigned long)GCAlloc::kFinalize);
int index = GetIndex(b, item);
GCAssert(index >= 0);
Clear4BitsAndSet(b, index, flags & kFinalize);
b->numItems++;
#ifdef MMGC_MEMORY_INFO
m_numAlloc++;
#endif
// If we're out of free items, be sure to remove ourselves from the
// list of blocks with free items. TODO Minor optimization: when we
// carve an item off the end of the block, we don't need to check here
// unless we just set b->nextItem to NULL.
if (b->IsFull()) {
m_firstFree = b->nextFree;
b->nextFree = NULL;
GCAssert(b->prevFree == NULL);
if (m_firstFree)
m_firstFree->prevFree = 0;
}
// prevent mid-collection (ie destructor) allocations on un-swept pages from
// getting swept. If the page is finalized and doesn't need sweeping we don't want
// to set the mark otherwise it will be marked when we start the next marking phase
// and write barriers won't fire (since its black)
if(m_gc->collecting)
{
if((b->finalizeState != m_gc->finalizedValue) || b->needsSweeping)
SetBit(b, index, kMark);
}
GCAssert((uintptr_t(item) & ~0xfff) == (uintptr_t) b);
GCAssert((uintptr_t(item) & 7) == 0);
#ifdef MMGC_HOOKS
GCHeap* heap = GCHeap::GetGCHeap();
if(heap->HooksEnabled())
{
size_t userSize = m_itemSize - DebugSize();
#ifdef MMGC_MEMORY_PROFILER
m_totalAskSize += size;
heap->AllocHook(GetUserPointer(item), size, userSize);
#else
heap->AllocHook(GetUserPointer(item), 0, userSize);
#endif
}
#endif
return item;
}
void* FixedAlloc::Alloc(size_t size, FixedMallocOpts opts)
{
(void)size;
GCAssertMsg(m_heap->StackEnteredCheck() || (opts&kCanFail) != 0, "MMGC_ENTER must be on the stack");
GCAssertMsg(((size_t)m_itemSize >= size), "allocator itemsize too small");
if(!m_firstFree) {
bool canFail = (opts & kCanFail) != 0;
CreateChunk(canFail);
if(!m_firstFree) {
if (!canFail) {
GCAssertMsg(0, "Memory allocation failed to abort properly");
GCHeap::SignalInconsistentHeapState("Failed to abort");
/*NOTREACHED*/
}
return NULL;
}
}
FixedBlock* b = m_firstFree;
GCAssert(b && !IsFull(b));
b->numAlloc++;
// Consume the free list if available
void *item = NULL;
if (b->firstFree) {
item = b->firstFree;
b->firstFree = *((void**)item);
// assert that the freelist hasn't been tampered with (by writing to the first 4 bytes)
GCAssert(b->firstFree == NULL ||
(b->firstFree >= b->items &&
(((uintptr_t)b->firstFree - (uintptr_t)b->items) % b->size) == 0 &&
(uintptr_t) b->firstFree < ((uintptr_t)b & ~0xfff) + GCHeap::kBlockSize));
#ifdef MMGC_MEMORY_INFO
//check for writes on deleted memory
VerifyFreeBlockIntegrity(item, b->size);
#endif
} else {
// Take next item from end of block
item = b->nextItem;
GCAssert(item != 0);
if(!IsFull(b)) {
// There are more items at the end of the block
b->nextItem = (void *) ((uintptr_t)item+m_itemSize);
} else {
b->nextItem = 0;
}
}
// If we're out of free items, be sure to remove ourselves from the
// list of blocks with free items.
if (IsFull(b)) {
m_firstFree = b->nextFree;
b->nextFree = NULL;
GCAssert(b->prevFree == NULL);
if (m_firstFree)
m_firstFree->prevFree = 0;
}
item = GetUserPointer(item);
#ifdef MMGC_HOOKS
if(m_heap->HooksEnabled())
{
#ifdef MMGC_MEMORY_PROFILER
m_totalAskSize += size;
#endif
m_heap->AllocHook(item, size, b->size - DebugSize());
}
#endif
#ifdef _DEBUG
// fresh memory poisoning
if((opts & kZero) == 0)
memset(item, 0xfa, b->size - DebugSize());
#endif
if((opts & kZero) != 0)
memset(item, 0, b->size - DebugSize());
return item;
}
