bool
SmallNormalHeapBucketBase<TBlockType>::RescanObjectsOnPage(TBlockType * block, char* pageAddress, char * blockStartAddress, BVStatic<TBlockAttributes::BitVectorCount> * heapBlockMarkBits, const uint localObjectSize, uint bucketIndex, __out_opt bool* anyObjectRescanned, Recycler * recycler)
{
RECYCLER_STATS_ADD(recycler, markData.rescanPageCount, TBlockAttributes::PageCount);
// By the time we get here, we should have ensured that there's a mark on any page somewhere.
// REVIEW: Worth check on just the page's mark bits?
Assert(!heapBlockMarkBits->IsAllClear());
if (anyObjectRescanned != nullptr)
{
*anyObjectRescanned = false;
}
Assert((char*)pageAddress - blockStartAddress < TBlockAttributes::PageCount * AutoSystemInfo::PageSize);
const uint pageByteOffset = static_cast<uint>((char*)pageAddress - blockStartAddress);
uint firstObjectOnPageIndex = pageByteOffset / localObjectSize;
// This is not necessarily the address on the first object that starts on the page
// If the last object on the previous page spans two pages, this is the address of that object
// We do it this way so that we can figure out if we need to rescan the first few bytes of the page
// if the actual first object on this page is not located at the start of the page
char* const startObjectAddress = blockStartAddress + (firstObjectOnPageIndex * localObjectSize);
const uint startBitIndex = TBlockType::GetAddressBitIndex(startObjectAddress);
const uint pageStartBitIndex = pageByteOffset >> HeapConstants::ObjectAllocationShift;
Assert(pageByteOffset / AutoSystemInfo::PageSize < USHRT_MAX);
const ushort pageNumber = static_cast<const ushort>(pageByteOffset / AutoSystemInfo::PageSize);
const typename TBlockType::BlockInfo& blockInfoForPage = HeapInfo::GetBlockInfo<TBlockAttributes>(localObjectSize)[pageNumber];
bool lastObjectOnPreviousPageMarked = false;
// Calculate the mark count here since we no longer keep track during marking
uint rescanMarkCount = TBlockType::CalculateMarkCountForPage(heapBlockMarkBits, bucketIndex, pageStartBitIndex);
const uint pageObjectCount = blockInfoForPage.pageObjectCount;
const uint localObjectCount = (TBlockAttributes::PageCount * AutoSystemInfo::PageSize) / localObjectSize;
// With protected unallocatable ending pages and reset writewatch, we should never be scanning on these pages.
if (firstObjectOnPageIndex >= localObjectCount)
{
ReportFatalException(NULL, E_FAIL, Fatal_Recycler_MemoryCorruption, 3);
}
// If all objects are marked, rescan whole block at once
if (TBlockType::CanRescanFullBlock() && rescanMarkCount == pageObjectCount)
{
// REVIEW: Can we optimize this more?
if (!recycler->AddMark(pageAddress, AutoSystemInfo::PageSize))
{
// Failed to add to the mark stack due to OOM.
return false;
}
RECYCLER_STATS_ADD(recycler, markData.rescanObjectCount, pageObjectCount);
RECYCLER_STATS_ADD(recycler, markData.rescanObjectByteCount, localObjectSize * pageObjectCount);
if (anyObjectRescanned != nullptr)
{
*anyObjectRescanned = true;
}
return true;
}
if (startObjectAddress != pageAddress)
{
// If the last object on the previous page that spans into the current page is marked,
// we need to count that in the markCount for rescan
Assert(startObjectAddress >= blockStartAddress && startObjectAddress < pageAddress);
lastObjectOnPreviousPageMarked = (heapBlockMarkBits->Test(startBitIndex) == TRUE);
if (lastObjectOnPreviousPageMarked)
{
rescanMarkCount++;
}
}
const uint objectBitDelta = SmallHeapBlockT<TBlockAttributes>::GetObjectBitDeltaForBucketIndex(bucketIndex);
uint rescanCount = 0;
uint objectIndex = firstObjectOnPageIndex;
for (uint bitIndex = startBitIndex; rescanCount < rescanMarkCount; objectIndex++, bitIndex += objectBitDelta)
{
Assert(objectIndex < localObjectCount);
Assert(!HeapInfo::GetInvalidBitVectorForBucket<TBlockAttributes>(bucketIndex)->Test(bitIndex));
if (heapBlockMarkBits->Test(bitIndex))
{
char * objectAddress = blockStartAddress + objectIndex * localObjectSize;
if (!TBlockType::RescanObject(block, objectAddress, localObjectSize, objectIndex, recycler))
{
// Failed to add to the mark stack due to OOM.
return false;
}
rescanCount++;
}
}
// Mark bits should not have changed during the Rescan
if (startObjectAddress != pageAddress && lastObjectOnPreviousPageMarked)
{
Assert(rescanMarkCount == TBlockType::CalculateMarkCountForPage(heapBlockMarkBits, bucketIndex, pageStartBitIndex) + 1);
}
else
{
Assert(rescanMarkCount == TBlockType::CalculateMarkCountForPage(heapBlockMarkBits, bucketIndex, pageStartBitIndex));
}
#if DBG
// We stopped when we hit the rescanMarkCount.
// Make sure no other objects were marked, otherwise our rescanMarkCount was wrong.
for (uint i = objectIndex + 1; i < blockInfoForPage.lastObjectIndexOnPage; i++)
{
Assert(!heapBlockMarkBits->Test(i * objectBitDelta));
}
#endif
// Let the caller know if we rescanned anything on this page
if (anyObjectRescanned != nullptr)
{
(*anyObjectRescanned) = (rescanCount > 0);
}
return true;
}
char*
LargeHeapBucket::PageHeapAlloc(Recycler * recycler, size_t sizeCat, size_t size, ObjectInfoBits attributes, PageHeapMode mode, bool nothrow)
{
Segment * segment;
size_t pageCount = LargeHeapBlock::GetPagesNeeded(size, false);
if (pageCount == 0)
{
if (nothrow == false)
{
// overflow
// Since nothrow is false here, it's okay to throw
recycler->OutOfMemory();
}
return nullptr;
}
if(size<sizeof(void*))
{
attributes = (ObjectInfoBits)(attributes | LeafBit);
}
size_t actualPageCount = pageCount + 1; // 1 for guard page
auto pageAllocator = recycler->GetRecyclerLargeBlockPageAllocator();
char * baseAddress = pageAllocator->Alloc(&actualPageCount, &segment);
if (baseAddress == nullptr)
{
return nullptr;
}
size_t guardPageCount = actualPageCount - pageCount; // pageAllocator can return more than asked pages
char* address = nullptr;
char* guardPageAddress = nullptr;
if (heapInfo->pageHeapMode == PageHeapMode::PageHeapModeBlockStart)
{
address = baseAddress + AutoSystemInfo::PageSize * guardPageCount;
guardPageAddress = baseAddress;
}
else if (heapInfo->pageHeapMode == PageHeapMode::PageHeapModeBlockEnd)
{
address = baseAddress;
guardPageAddress = baseAddress + pageCount * AutoSystemInfo::PageSize;
}
else
{
AnalysisAssert(false);
}
LargeHeapBlock * heapBlock = LargeHeapBlock::New(address, pageCount, segment, 1, nullptr);
if (!heapBlock)
{
pageAllocator->SuspendIdleDecommit();
pageAllocator->Release(baseAddress, actualPageCount, segment);
pageAllocator->ResumeIdleDecommit();
return nullptr;
}
heapBlock->heapInfo = this->heapInfo;
heapBlock->actualPageCount = actualPageCount;
heapBlock->guardPageAddress = guardPageAddress;
// fill pattern before set pageHeapMode, so background scan stack may verify the pattern
size_t usedSpace = sizeof(LargeObjectHeader) + size;
memset(address + usedSpace, 0xF0, pageCount * AutoSystemInfo::PageSize - usedSpace);
heapBlock->pageHeapMode = heapInfo->pageHeapMode;
if (!recycler->heapBlockMap.SetHeapBlock(address, pageCount, heapBlock, HeapBlock::HeapBlockType::LargeBlockType, 0))
{
pageAllocator->SuspendIdleDecommit();
heapBlock->ReleasePages(recycler);
pageAllocator->ResumeIdleDecommit();
LargeHeapBlock::Delete(heapBlock);
return nullptr;
}
heapBlock->ResetMarks(ResetMarkFlags_None, recycler);
char * memBlock = heapBlock->Alloc(size, attributes);
Assert(memBlock != nullptr);
#pragma prefast(suppress:6250, "This method decommits memory")
if (::VirtualFree(guardPageAddress, AutoSystemInfo::PageSize * guardPageCount, MEM_DECOMMIT) == FALSE)
{
AssertMsg(false, "Unable to decommit guard page.");
ReportFatalException(NULL, E_FAIL, Fatal_Internal_Error, 2);
return nullptr;
}
if (this->largePageHeapBlockList)
{
HeapBlockList::Tail(this->largePageHeapBlockList)->SetNextBlock(heapBlock);
}
else
{
this->largePageHeapBlockList = heapBlock;
}
#if ENABLE_PARTIAL_GC
recycler->autoHeap.uncollectedNewPageCount += pageCount;
#endif
RECYCLER_SLOW_CHECK(this->heapInfo->heapBlockCount[HeapBlock::HeapBlockType::LargeBlockType]++);
RECYCLER_PERF_COUNTER_ADD(FreeObjectSize, heapBlock->GetPageCount() * AutoSystemInfo::PageSize);
if (recycler->ShouldCapturePageHeapAllocStack())
{
heapBlock->CapturePageHeapAllocStack();
}
return memBlock;
}
_NOINLINE void DebugHeap_OOM_fatal_error()
{
int scenario = 3;
ReportFatalException(NULL, E_OUTOFMEMORY, Fatal_Debug_Heap_OUTOFMEMORY, scenario);
}
_NOINLINE void Version_Inconsistency_fatal_error()
{
int scenario = 4;
ReportFatalException(NULL, E_UNEXPECTED, Fatal_Version_Inconsistency, scenario);
}
_NOINLINE void FailedToBox_OOM_fatal_error(
__in ULONG_PTR context)
{
int scenario = 1;
ReportFatalException(context, E_UNEXPECTED, Fatal_FailedToBox_OUTOFMEMORY, scenario);
}
_NOINLINE void X64WriteBarrier_OOM_fatal_error()
{
int scenario = 3;
ReportFatalException(NULL, E_OUTOFMEMORY, WriteBarrier_OUTOFMEMORY, scenario);
}
_NOINLINE void MarkStack_OOM_fatal_error()
{
int scenario = 1;
ReportFatalException(NULL, E_OUTOFMEMORY, MarkStack_OUTOFMEMORY, scenario);
};
_NOINLINE void Amd64StackWalkerOutOfContexts_fatal_error(
__in ULONG_PTR context)
{
int scenario = 1;
ReportFatalException(context, E_UNEXPECTED, Fatal_Amd64StackWalkerOutOfContexts, scenario);
}
_NOINLINE void JavascriptDispatch_OOM_fatal_error(
__in ULONG_PTR context)
{
int scenario = 1;
ReportFatalException(context, E_OUTOFMEMORY, JavascriptDispatch_OUTOFMEMORY, scenario);
};
_NOINLINE void CustomHeap_BadPageState_fatal_error(
__in ULONG_PTR context)
{
int scenario = 1;
ReportFatalException(context, E_UNEXPECTED, CustomHeap_MEMORYCORRUPTION, scenario);
};
_NOINLINE void UnexpectedExceptionHandling_fatal_error(EXCEPTION_POINTERS * originalException)
{
int scenario = 7;
ReportFatalException(NULL, E_UNEXPECTED, Fatal_UnexpectedExceptionHandling, scenario);
}
_NOINLINE void EntryExitRecord_Corrupted_fatal_error()
{
int scenario = 6;
ReportFatalException(NULL, E_UNEXPECTED, Fatal_EntryExitRecordCorruption, scenario);
}
_NOINLINE void Debugger_AttachDetach_fatal_error(HRESULT hr)
{
int scenario = 5;
ReportFatalException(NULL, hr, Fatal_Debugger_AttachDetach_Failure, scenario);
}
_NOINLINE void FromDOM_NoScriptScope_fatal_error()
{
int scenario = 5;
ReportFatalException(NULL, E_UNEXPECTED, EnterScript_FromDOM_NoScriptScope, scenario);
}
_NOINLINE void LargeHeapBlock_Metadata_Corrupted(
__in ULONG_PTR context, __in unsigned char calculatedChecksum)
{
int scenario = calculatedChecksum; /* For debugging purpose if checksum mismatch happen*/
ReportFatalException(context, E_UNEXPECTED, LargeHeapBlock_Metadata_Corrupt, scenario);
};
