//=====================================================================================================
// Free
//=====================================================================================================
void
LargeHeapBucket::ExplicitFree(void * object, size_t sizeCat)
{
Assert(HeapInfo::GetMediumObjectAlignedSizeNoCheck(sizeCat) == this->sizeCat);
LargeObjectHeader * header = LargeHeapBlock::GetHeaderFromAddress(object);
Assert(header->GetAttributes(this->heapInfo->recycler->Cookie) == ObjectInfoBits::NoBit || header->GetAttributes(this->heapInfo->recycler->Cookie) == ObjectInfoBits::LeafBit);
Assert(!header->isExplicitFreed);
DebugOnly(header->isExplicitFreed = true);
Assert(header->objectSize >= sizeCat);
#if DBG
HeapBlock* heapBlock = this->GetRecycler()->FindHeapBlock(object);
Assert(heapBlock != nullptr);
Assert(heapBlock->IsLargeHeapBlock());
LargeHeapBlock * largeHeapBlock = (LargeHeapBlock *)heapBlock;
LargeObjectHeader * dbgHeader;
Assert(largeHeapBlock->GetObjectHeader(object, &dbgHeader));
Assert(dbgHeader == header);
#endif
FreeObject * freeObject = (FreeObject *)object;
freeObject->SetNext(this->explicitFreeList);
this->explicitFreeList = freeObject;
header->SetAttributes(this->heapInfo->recycler->Cookie, ObjectInfoBits::LeafBit); // We can stop scanning it now.
}
char *
LargeHeapBucket::TryAllocFromExplicitFreeList(Recycler * recycler, size_t sizeCat, ObjectInfoBits attributes)
{
Assert((attributes & InternalObjectInfoBitMask) == attributes);
FreeObject * currFreeObject = this->explicitFreeList;
FreeObject * prevFreeObject = nullptr;
while (currFreeObject != nullptr)
{
char * memBlock = (char *)currFreeObject;
LargeObjectHeader * header = LargeHeapBlock::GetHeaderFromAddress(memBlock);
Assert(header->isExplicitFreed);
Assert(HeapInfo::GetMediumObjectAlignedSizeNoCheck(header->objectSize) == this->sizeCat);
if (header->objectSize < sizeCat)
{
prevFreeObject = currFreeObject;
currFreeObject = currFreeObject->GetNext();
continue;
}
DebugOnly(header->isExplicitFreed = false);
if (prevFreeObject)
{
prevFreeObject->SetNext(currFreeObject->GetNext());
}
else
{
this->explicitFreeList = currFreeObject->GetNext();
}
#ifdef RECYCLER_MEMORY_VERIFY
HeapBlock* heapBlock = recycler->FindHeapBlock(memBlock);
Assert(heapBlock != nullptr);
Assert(heapBlock->IsLargeHeapBlock());
LargeHeapBlock * largeHeapBlock = (LargeHeapBlock *)heapBlock;
LargeObjectHeader * dbgHeader;
Assert(largeHeapBlock->GetObjectHeader(memBlock, &dbgHeader));
Assert(dbgHeader == header);
((FreeObject *)memBlock)->DebugFillNext();
#endif
#ifdef RECYCLER_ZERO_MEM_CHECK
// TODO: large heap block doesn't separate leaf object on to different page allocator.
// so all the memory should still be zeroed.
memset(memBlock, 0, sizeof(FreeObject));
#endif
header->SetAttributes(recycler->Cookie, (attributes & StoredObjectInfoBitMask));
if ((attributes & ObjectInfoBits::FinalizeBit) != 0)
{
LargeHeapBlock* heapBlock = (LargeHeapBlock *)recycler->FindHeapBlock(memBlock);
heapBlock->finalizeCount++;
#ifdef RECYCLER_FINALIZE_CHECK
heapInfo->liveFinalizableObjectCount++;
heapInfo->newFinalizableObjectCount++;
#endif
}
return memBlock;
}
return nullptr;
}
void
SmallHeapBlockAllocator<TBlockType>::Clear()
{
TBlockType * heapBlock = this->heapBlock;
if (heapBlock != nullptr)
{
Assert(heapBlock->isInAllocator);
heapBlock->isInAllocator = false;
FreeObject * remainingFreeObjectList = nullptr;
if (this->endAddress != nullptr)
{
#ifdef RECYCLER_TRACK_NATIVE_ALLOCATED_OBJECTS
TrackNativeAllocatedObjects();
lastNonNativeBumpAllocatedBlock = nullptr;
#endif
#ifdef PROFILE_RECYCLER_ALLOC
// Need to tell the tracker
this->bucket->heapInfo->recycler->TrackUnallocated((char *)this->freeObjectList, this->endAddress, this->bucket->sizeCat);
#endif
RecyclerMemoryTracking::ReportUnallocated(this->heapBlock->heapBucket->heapInfo->recycler, (char *)this->freeObjectList, this->endAddress, heapBlock->heapBucket->sizeCat);
#ifdef RECYCLER_PERF_COUNTERS
size_t unallocatedObjects = heapBlock->objectCount - ((char *)this->freeObjectList - heapBlock->address) / heapBlock->objectSize;
size_t unallocatedObjectBytes = unallocatedObjects * heapBlock->GetObjectSize();
RECYCLER_PERF_COUNTER_ADD(LiveObject, unallocatedObjects);
RECYCLER_PERF_COUNTER_ADD(LiveObjectSize, unallocatedObjectBytes);
RECYCLER_PERF_COUNTER_SUB(FreeObjectSize, unallocatedObjectBytes);
RECYCLER_PERF_COUNTER_ADD(SmallHeapBlockLiveObject, unallocatedObjects);
RECYCLER_PERF_COUNTER_ADD(SmallHeapBlockLiveObjectSize, unallocatedObjectBytes);
RECYCLER_PERF_COUNTER_SUB(SmallHeapBlockFreeObjectSize, unallocatedObjectBytes);
#endif
Assert(heapBlock->freeObjectList == nullptr);
this->endAddress = nullptr;
}
else
{
remainingFreeObjectList = this->freeObjectList;
heapBlock->freeObjectList = remainingFreeObjectList;
}
this->freeObjectList = nullptr;
// this->freeObjectList and this->lastFreeCount are accessed in SmallHeapBlock::ResetMarks
// the order of access there is first we see if lastFreeCount = 0, and if it is, we assert
// that freeObjectList = null. Because of ARM's memory model, we need to insert barriers
// so that the two variables can be accessed correctly across threads. Here, after we write
// to this->freeObjectList, we insert a write barrier so that if this->lastFreeCount is 0,
// this->freeObjectList must have been set to null. On the other end, we stick a read barrier
// We use the MemoryBarrier macro because of ARMs lack of a separate read barrier
#if defined(_M_ARM32_OR_ARM64)
#if DBG
MemoryBarrier();
#endif
#endif
if (remainingFreeObjectList == nullptr)
{
uint lastFreeCount = heapBlock->GetAndClearLastFreeCount();
heapBlock->heapBucket->heapInfo->uncollectedAllocBytes += lastFreeCount * heapBlock->GetObjectSize();
Assert(heapBlock->lastUncollectedAllocBytes == 0);
DebugOnly(heapBlock->lastUncollectedAllocBytes = lastFreeCount * heapBlock->GetObjectSize());
}
else
{
DebugOnly(heapBlock->SetIsClearedFromAllocator(true));
}
this->heapBlock = nullptr;
RECYCLER_SLOW_CHECK(heapBlock->CheckDebugFreeBitVector(false));
}
else if (this->freeObjectList != nullptr)
{
// Explicit Free Object List
#ifdef RECYCLER_MEMORY_VERIFY
FreeObject* freeObject = this->freeObjectList;
while (freeObject)
{
HeapBlock* heapBlock = this->bucket->GetRecycler()->FindHeapBlock((void*) freeObject);
Assert(heapBlock != nullptr);
Assert(!heapBlock->IsLargeHeapBlock());
TBlockType* smallBlock = (TBlockType*)heapBlock;
smallBlock->ClearExplicitFreeBitForObject((void*) freeObject);
freeObject = freeObject->GetNext();
}
#endif
this->freeObjectList = nullptr;
}
}