void
LargeHeapBucket::ConcurrentTransferSweptObjects(RecyclerSweep& recyclerSweep)
{
#if ENABLE_PARTIAL_GC
Assert(!recyclerSweep.InPartialCollectMode());
#endif
Assert(!recyclerSweep.IsBackground());
HeapBlockList::ForEachEditing(this->pendingSweepLargeBlockList, [this](LargeHeapBlock * heapBlock)
{
heapBlock->TransferSweptObjects();
ReinsertLargeHeapBlock(heapBlock);
});
this->pendingSweepLargeBlockList = nullptr;
#if ENABLE_PARTIAL_GC
// If we did a background finish partial collect, we have left the partialSweptLargeBlockList
// there because can't reinsert the heap block in the background, do it here now.
HeapBlockList::ForEachEditing(this->partialSweptLargeBlockList, [this](LargeHeapBlock * heapBlock)
{
ReinsertLargeHeapBlock(heapBlock);
});
this->partialSweptLargeBlockList = nullptr;
#endif
}
void
LargeHeapBucket::SweepPendingObjects(RecyclerSweep& recyclerSweep)
{
#if ENABLE_CONCURRENT_GC
if (recyclerSweep.IsBackground())
{
Recycler * recycler = recyclerSweep.GetRecycler();
#if ENABLE_PARTIAL_GC
if (recycler->inPartialCollectMode)
{
HeapBlockList::ForEach(this->pendingSweepLargeBlockList, [recycler](LargeHeapBlock * heapBlock)
{
// Page heap blocks are never swept concurrently
heapBlock->SweepObjects<SweepMode_ConcurrentPartial>(recycler);
});
}
else
#endif
{
HeapBlockList::ForEach(this->pendingSweepLargeBlockList, [recycler](LargeHeapBlock * heapBlock)
{
// Page heap blocks are never swept concurrently
heapBlock->SweepObjects<SweepMode_Concurrent>(recycler);
});
}
}
else
{
Assert(this->pendingSweepLargeBlockList == nullptr);
}
#endif
}
void
SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& recyclerSweep, TBlockType * heapBlockList,
TBlockType *& reuseBlocklist, TBlockType *&unusedBlockList, Fn callback)
{
HeapBlockList::ForEachEditing(heapBlockList,
[&recyclerSweep, &reuseBlocklist, &unusedBlockList, callback](TBlockType * heapBlock)
{
uint expectFreeByteCount;
if (heapBlock->DoPartialReusePage(recyclerSweep, expectFreeByteCount))
{
callback(heapBlock, true);
// Reuse the page
heapBlock->SetNextBlock(reuseBlocklist);
reuseBlocklist = heapBlock;
RECYCLER_STATS_ADD(recyclerSweep.GetRecycler(), smallNonLeafHeapBlockPartialReuseBytes[heapBlock->GetHeapBlockType()], expectFreeByteCount);
RECYCLER_STATS_INC(recyclerSweep.GetRecycler(), smallNonLeafHeapBlockPartialReuseCount[heapBlock->GetHeapBlockType()]);
}
else
{
// Don't not reuse the page if it don't have much free memory.
callback(heapBlock, false);
heapBlock->SetNextBlock(unusedBlockList);
unusedBlockList = heapBlock;
recyclerSweep.AddUnusedFreeByteCount(expectFreeByteCount);
RECYCLER_STATS_ADD(recyclerSweep.GetRecycler(), smallNonLeafHeapBlockPartialUnusedBytes[heapBlock->GetHeapBlockType()], expectFreeByteCount);
RECYCLER_STATS_INC(recyclerSweep.GetRecycler(), smallNonLeafHeapBlockPartialUnusedCount[heapBlock->GetHeapBlockType()]);
}
});
}
void
LargeHeapBucket::SweepLargeHeapBlockList(RecyclerSweep& recyclerSweep, LargeHeapBlock * heapBlockList)
{
Recycler * recycler = recyclerSweep.GetRecycler();
HeapBlockList::ForEachEditing(heapBlockList, [this, &recyclerSweep, recycler](LargeHeapBlock * heapBlock)
{
this->UnregisterFreeList(heapBlock->GetFreeList());
// CONCURRENT-TODO: Allow large block to be sweep in the background
SweepState state = heapBlock->Sweep(recyclerSweep, false);
// If the block is already in the pending dispose list (re-entrant GC scenario), do nothing, leave it there
if (heapBlock->IsInPendingDisposeList()) return;
switch (state)
{
case SweepStateEmpty:
heapBlock->ReleasePagesSweep(recycler);
LargeHeapBlock::Delete(heapBlock);
RECYCLER_SLOW_CHECK(this->heapInfo->heapBlockCount[HeapBlock::HeapBlockType::LargeBlockType]--);
break;
case SweepStateFull:
heapBlock->SetNextBlock(this->fullLargeBlockList);
this->fullLargeBlockList = heapBlock;
break;
case SweepStateSwept:
if (supportFreeList)
{
ConstructFreelist(heapBlock);
}
else
{
ReinsertLargeHeapBlock(heapBlock);
}
break;
case SweepStatePendingDispose:
Assert(!recyclerSweep.IsBackground());
Assert(!recycler->hasPendingTransferDisposedObjects);
heapBlock->SetNextBlock(this->pendingDisposeLargeBlockList);
this->pendingDisposeLargeBlockList = heapBlock;
heapBlock->SetIsInPendingDisposeList(true);
#if DBG
heapBlock->SetHasDisposeBeenCalled(false);
#endif
recycler->hasDisposableObject = true;
break;
#if ENABLE_CONCURRENT_GC
case SweepStatePendingSweep:
heapBlock->SetNextBlock(this->pendingSweepLargeBlockList);
this->pendingSweepLargeBlockList = heapBlock;
break;
#endif
}
});
}
void
SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(RecyclerSweep& recyclerSweep)
{
RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(recyclerSweep.IsBackground()));
Assert(this->GetRecycler()->inPartialCollectMode);
TBlockType * currentHeapBlockList = this->heapBlockList;
this->heapBlockList = nullptr;
SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(recyclerSweep, currentHeapBlockList, this->heapBlockList,
this->partialHeapBlockList,
[](TBlockType * heapBlock, bool isReused) {});
#if ENABLE_CONCURRENT_GC
// only collect data for pending sweep list but don't sweep yet
// until we have adjusted the heuristics, and SweepPartialReusePages will
// sweep the page that we are going to reuse in thread.
TBlockType *& pendingSweepList = recyclerSweep.GetPendingSweepBlockList(this);
currentHeapBlockList = pendingSweepList;
pendingSweepList = nullptr;
Recycler * recycler = recyclerSweep.GetRecycler();
SmallNormalHeapBucketBase<TBlockType>::SweepPartialReusePages(recyclerSweep, currentHeapBlockList, this->heapBlockList,
pendingSweepList,
[recycler](TBlockType * heapBlock, bool isReused)
{
if (isReused)
{
// Finalizable blocks are always swept in thread, so shouldn't be here
Assert(!heapBlock->IsAnyFinalizableBlock());
// Page heap blocks are never swept concurrently
heapBlock->template SweepObjects<SweepMode_InThread>(recycler);
// This block has been counted as concurrently swept, and now we changed our mind
// and sweep it in thread. Remove the count
RECYCLER_STATS_DEC(recycler, heapBlockConcurrentSweptCount[heapBlock->GetHeapBlockType()]);
}
}
);
#endif
RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(recyclerSweep.IsBackground()));
this->StartAllocationAfterSweep();
// PARTIALGC-TODO: revisit partial heap blocks to see if they can be put back into use
// since the heuristics limit may be been changed.
}
void
SmallNormalHeapBucketBase<TBlockType>::Sweep(RecyclerSweep& recyclerSweep)
{
#if ENABLE_PARTIAL_GC
#if DBG
Recycler * recycler = recyclerSweep.GetRecycler();
// Don't need sweep the partialHeapBlockList, the partially collected heap block list.
// There should be nothing there that is free-able since the last time we swept
Assert(recyclerSweep.InPartialCollect() || partialHeapBlockList == nullptr);
#if ENABLE_CONCURRENT_GC
Assert(recyclerSweep.InPartialCollect() || partialSweptHeapBlockList == nullptr);
#endif
this->SweepVerifyPartialBlocks(recycler, this->partialHeapBlockList);
#endif
#endif
BaseT::SweepBucket(recyclerSweep, [](RecyclerSweep& recyclerSweep){});
}
void
LargeHeapBucket::ConcurrentPartialTransferSweptObjects(RecyclerSweep& recyclerSweep)
{
Assert(recyclerSweep.InPartialCollectMode());
Assert(!recyclerSweep.IsBackground());
RECYCLER_SLOW_CHECK(this->VerifyLargeHeapBlockCount());
LargeHeapBlock * list = this->pendingSweepLargeBlockList;
this->pendingSweepLargeBlockList = nullptr;
HeapBlockList::ForEachEditing(list, [this](LargeHeapBlock * heapBlock)
{
// GC-REVIEW: We could maybe reuse the large objects
heapBlock->PartialTransferSweptObjects();
heapBlock->SetNextBlock(this->partialSweptLargeBlockList);
this->partialSweptLargeBlockList = heapBlock;
});
RECYCLER_SLOW_CHECK(this->VerifyLargeHeapBlockCount());
}
void
LargeHeapBucket::Sweep(RecyclerSweep& recyclerSweep)
{
#if ENABLE_CONCURRENT_GC
// CONCURRENT-TODO: large buckets are not swept in the background currently.
Assert(!recyclerSweep.GetRecycler()->IsConcurrentExecutingState());
#endif
LargeHeapBlock * currentLargeObjectBlocks = largeBlockList;
#ifdef RECYCLER_PAGE_HEAP
LargeHeapBlock * currentLargePageHeapObjectBlocks = largePageHeapBlockList;
#endif
LargeHeapBlock * currentFullLargeObjectBlocks = fullLargeBlockList;
LargeHeapBlock * currentDisposeLargeBlockList = pendingDisposeLargeBlockList;
this->largeBlockList = nullptr;
#ifdef RECYCLER_PAGE_HEAP
this->largePageHeapBlockList = nullptr;
#endif
this->fullLargeBlockList = nullptr;
// Clear the free list before sweep
// We'll reconstruct the free list during sweep
if (this->supportFreeList)
{
#if DBG
LargeAllocationVerboseTrace(recyclerSweep.GetRecycler()->GetRecyclerFlagsTable(), _u("Resetting free list for 0x%x bucket\n"), this->sizeCat);
#endif
this->freeList = nullptr;
this->explicitFreeList = nullptr;
}
#if ENABLE_CONCURRENT_GC
Assert(this->pendingSweepLargeBlockList == nullptr);
#endif
SweepLargeHeapBlockList(recyclerSweep, currentLargeObjectBlocks);
#ifdef RECYCLER_PAGE_HEAP
SweepLargeHeapBlockList(recyclerSweep, currentLargePageHeapObjectBlocks);
#endif
SweepLargeHeapBlockList(recyclerSweep, currentFullLargeObjectBlocks);
SweepLargeHeapBlockList(recyclerSweep, currentDisposeLargeBlockList);
}
void
SmallFinalizableHeapBucketBaseT<TBlockType>::Sweep(RecyclerSweep& recyclerSweep)
{
Assert(!recyclerSweep.IsBackground());
#if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
Assert(this->tempPendingDisposeList == nullptr);
this->tempPendingDisposeList = pendingDisposeList;
#endif
TBlockType * currentDisposeList = pendingDisposeList;
this->pendingDisposeList = nullptr;
BaseT::SweepBucket(recyclerSweep, [=](RecyclerSweep& recyclerSweep)
{
#if DBG
if (TBlockType::HeapBlockAttributes::IsSmallBlock)
{
recyclerSweep.SetupVerifyListConsistencyDataForSmallBlock(nullptr, false, true);
}
else if (TBlockType::HeapBlockAttributes::IsMediumBlock)
{
recyclerSweep.SetupVerifyListConsistencyDataForMediumBlock(nullptr, false, true);
}
else
{
Assert(false);
}
#endif
HeapBucketT<TBlockType>::SweepHeapBlockList(recyclerSweep, currentDisposeList, false);
#if DBG || defined(RECYCLER_SLOW_CHECK_ENABLED)
Assert(this->tempPendingDisposeList == currentDisposeList);
this->tempPendingDisposeList = nullptr;
#endif
RECYCLER_SLOW_CHECK(this->VerifyHeapBlockCount(recyclerSweep.IsBackground()));
});
}
void
SmallNormalHeapBucketBase<TBlockType>::SweepPendingObjects(RecyclerSweep& recyclerSweep)
{
RECYCLER_SLOW_CHECK(VerifyHeapBlockCount(recyclerSweep.IsBackground()));
CompileAssert(!BaseT::IsLeafBucket);
TBlockType *& pendingSweepList = recyclerSweep.GetPendingSweepBlockList(this);
TBlockType * const list = pendingSweepList;
Recycler * const recycler = recyclerSweep.GetRecycler();
#if ENABLE_PARTIAL_GC
bool const partialSweep = recycler->inPartialCollectMode;
#endif
if (list)
{
pendingSweepList = nullptr;
#if ENABLE_PARTIAL_GC
if (partialSweep)
{
// We did a partial sweep.
// Blocks in the pendingSweepList are the ones we decided not to reuse.
HeapBlockList::ForEachEditing(list, [this, recycler](TBlockType * heapBlock)
{
// We are not going to reuse this block.
// SweepMode_ConcurrentPartial will not actually collect anything, it will just update some state.
// The sweepable objects will be collected in a future Sweep.
// Note, page heap blocks are never swept concurrently
heapBlock->template SweepObjects<SweepMode_ConcurrentPartial>(recycler);
// page heap mode should never reach here, so don't check pageheap enabled or not
if (heapBlock->template HasFreeObject<false>())
{
// We have pre-existing free objects, so put this in the partialSweptHeapBlockList
heapBlock->SetNextBlock(this->partialSweptHeapBlockList);
this->partialSweptHeapBlockList = heapBlock;
}
else
{
// No free objects, so put in the fullBlockList
heapBlock->SetNextBlock(this->fullBlockList);
this->fullBlockList = heapBlock;
}
});
}
else
#endif
{
// We decided not to do a partial sweep.
// Blocks in the pendingSweepList need to have a regular sweep.
TBlockType * tail = SweepPendingObjects<SweepMode_Concurrent>(recycler, list);
tail->SetNextBlock(this->heapBlockList);
this->heapBlockList = list;
this->StartAllocationAfterSweep();
}
RECYCLER_SLOW_CHECK(VerifyHeapBlockCount(recyclerSweep.IsBackground()));
}
Assert(!this->IsAllocationStopped());
}
