char * LargeHeapBucket::TryAllocFromFreeList(Recycler * recycler, size_t sizeCat, ObjectInfoBits attributes) { Assert((attributes & InternalObjectInfoBitMask) == attributes); LargeHeapBlockFreeList* freeListEntry = this->freeList; // Walk through the free list, find the first entry that can fit our desired size while (freeListEntry) { LargeHeapBlock* heapBlock = freeListEntry->heapBlock; char * memBlock = heapBlock->TryAllocFromFreeList(sizeCat, attributes); if (memBlock) { // Don't need to verify zero fill here since we will do it in LargeHeapBucket::Alloc return memBlock; } else { #if DBG LargeAllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("Unable to allocate object of size 0x%x from freelist\n"), sizeCat); #endif } freeListEntry = freeListEntry->next; } return nullptr; }
void LargeHeapBucket::Sweep(RecyclerSweep& recyclerSweep) { #if ENABLE_CONCURRENT_GC // CONCURRENT-TODO: large buckets are not swept in the background currently. Assert(!recyclerSweep.GetRecycler()->IsConcurrentExecutingState()); #endif LargeHeapBlock * currentLargeObjectBlocks = largeBlockList; #ifdef RECYCLER_PAGE_HEAP LargeHeapBlock * currentLargePageHeapObjectBlocks = largePageHeapBlockList; #endif LargeHeapBlock * currentFullLargeObjectBlocks = fullLargeBlockList; LargeHeapBlock * currentDisposeLargeBlockList = pendingDisposeLargeBlockList; this->largeBlockList = nullptr; #ifdef RECYCLER_PAGE_HEAP this->largePageHeapBlockList = nullptr; #endif this->fullLargeBlockList = nullptr; // Clear the free list before sweep // We'll reconstruct the free list during sweep if (this->supportFreeList) { #if DBG LargeAllocationVerboseTrace(recyclerSweep.GetRecycler()->GetRecyclerFlagsTable(), _u("Resetting free list for 0x%x bucket\n"), this->sizeCat); #endif this->freeList = nullptr; this->explicitFreeList = nullptr; } #if ENABLE_CONCURRENT_GC Assert(this->pendingSweepLargeBlockList == nullptr); #endif SweepLargeHeapBlockList(recyclerSweep, currentLargeObjectBlocks); #ifdef RECYCLER_PAGE_HEAP SweepLargeHeapBlockList(recyclerSweep, currentLargePageHeapObjectBlocks); #endif SweepLargeHeapBlockList(recyclerSweep, currentFullLargeObjectBlocks); SweepLargeHeapBlockList(recyclerSweep, currentDisposeLargeBlockList); }
void LargeHeapBucket::ConstructFreelist(LargeHeapBlock * heapBlock) { Assert(!heapBlock->hasPartialFreeObjects); Assert(!heapBlock->IsInPendingDisposeList()); // The free list is the only way we reuse heap block entries // so if the heap block is allocated from directly, it'll not // invalidate the free list LargeHeapBlockFreeList* freeList = heapBlock->GetFreeList(); Assert(freeList); if (freeList->entries) { this->RegisterFreeList(freeList); #if DBG LargeAllocationVerboseTrace(this->GetRecycler()->GetRecyclerFlagsTable(), _u("Free list created for 0x%x bucket\n"), this->sizeCat); #endif } ReinsertLargeHeapBlock(heapBlock); }
LargeHeapBlock* LargeHeapBucket::AddLargeHeapBlock(size_t size, bool nothrow) { Recycler* recycler = this->heapInfo->recycler; Segment * segment; size_t pageCount = LargeHeapBlock::GetPagesNeeded(size, this->supportFreeList); if (pageCount == 0) { if (nothrow == false) { // overflow // Since nothrow is false here, it's okay to throw recycler->OutOfMemory(); } return nullptr; } char * address = nullptr; size_t realPageCount = pageCount; address = recycler->GetRecyclerLargeBlockPageAllocator()->Alloc(&realPageCount, &segment); pageCount = realPageCount; if (address == nullptr) { return nullptr; } #ifdef RECYCLER_ZERO_MEM_CHECK recycler->VerifyZeroFill(address, pageCount * AutoSystemInfo::PageSize); #endif uint objectCount = LargeHeapBlock::GetMaxLargeObjectCount(pageCount, size); LargeHeapBlock * heapBlock = LargeHeapBlock::New(address, pageCount, segment, objectCount, supportFreeList ? this : nullptr); #if DBG LargeAllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("Allocated new large heap block 0x%p for sizeCat 0x%x\n"), heapBlock, sizeCat); #endif #ifdef ENABLE_JS_ETW #if ENABLE_DEBUG_CONFIG_OPTIONS if (segment->GetPageCount() > recycler->GetRecyclerLargeBlockPageAllocator()->GetMaxAllocPageCount()) { EventWriteJSCRIPT_INTERNAL_RECYCLER_EXTRALARGE_OBJECT_ALLOC(size); } #endif #endif if (!heapBlock) { recycler->GetRecyclerLargeBlockPageAllocator()->SuspendIdleDecommit(); recycler->GetRecyclerLargeBlockPageAllocator()->Release(address, pageCount, segment); recycler->GetRecyclerLargeBlockPageAllocator()->ResumeIdleDecommit(); return nullptr; } #if ENABLE_PARTIAL_GC recycler->autoHeap.uncollectedNewPageCount += pageCount; #endif RECYCLER_SLOW_CHECK(this->heapInfo->heapBlockCount[HeapBlock::HeapBlockType::LargeBlockType]++); heapBlock->heapInfo = this->heapInfo; heapBlock->lastCollectAllocCount = 0; Assert(recycler->collectionState != CollectionStateMark); if (!recycler->heapBlockMap.SetHeapBlock(address, pageCount, heapBlock, HeapBlock::HeapBlockType::LargeBlockType, 0)) { recycler->GetRecyclerLargeBlockPageAllocator()->SuspendIdleDecommit(); heapBlock->ReleasePages(recycler); recycler->GetRecyclerLargeBlockPageAllocator()->ResumeIdleDecommit(); LargeHeapBlock::Delete(heapBlock); RECYCLER_SLOW_CHECK(this->heapInfo->heapBlockCount[HeapBlock::HeapBlockType::LargeBlockType]--); return nullptr; } heapBlock->SetNextBlock(this->largeBlockList); this->largeBlockList = heapBlock; RECYCLER_PERF_COUNTER_ADD(FreeObjectSize, heapBlock->GetPageCount() * AutoSystemInfo::PageSize); return heapBlock; }
char* LargeHeapBucket::PageHeapAlloc(Recycler * recycler, size_t size, ObjectInfoBits attributes, PageHeapMode mode, bool nothrow) { size_t sizeCat = HeapInfo::GetAlignedSizeNoCheck(size); Segment * segment; size_t pageCount = LargeHeapBlock::GetPagesNeeded(size, this->supportFreeList); if (pageCount == 0) { if (nothrow == false) { // overflow // Since nothrow is false here, it's okay to throw recycler->OutOfMemory(); } return nullptr; } size_t actualPageCount = pageCount + 1; // for page heap char * baseAddress = recycler->GetRecyclerLargeBlockPageAllocator()->Alloc(&actualPageCount, &segment); if (baseAddress == nullptr) { return nullptr; } char* address = nullptr; char* guardPageAddress = nullptr; DWORD guardPageOldProtectFlags = PAGE_NOACCESS; if (heapInfo->pageHeapMode == PageHeapMode::PageHeapModeBlockStart) { address = baseAddress + AutoSystemInfo::PageSize; guardPageAddress = baseAddress; } else if (heapInfo->pageHeapMode == PageHeapMode::PageHeapModeBlockEnd) { address = baseAddress; guardPageAddress = baseAddress + pageCount* AutoSystemInfo::PageSize; } else { AnalysisAssert(false); } if (::VirtualProtect(static_cast<LPVOID>(guardPageAddress), AutoSystemInfo::PageSize, PAGE_NOACCESS, &guardPageOldProtectFlags) == FALSE) { AssertMsg(false, "Unable to set permission for guard page."); return nullptr; } #ifdef RECYCLER_ZERO_MEM_CHECK recycler->VerifyZeroFill(address, pageCount * AutoSystemInfo::PageSize); #endif LargeHeapBlock * heapBlock = LargeHeapBlock::New(address, pageCount, segment, 1, nullptr); if (!heapBlock) { recycler->GetRecyclerLargeBlockPageAllocator()->SuspendIdleDecommit(); recycler->GetRecyclerLargeBlockPageAllocator()->Release(address, actualPageCount, segment); recycler->GetRecyclerLargeBlockPageAllocator()->ResumeIdleDecommit(); return nullptr; } heapBlock->actualPageCount = actualPageCount; heapBlock->guardPageAddress = guardPageAddress; heapBlock->guardPageOldProtectFlags = guardPageOldProtectFlags; heapBlock->pageHeapMode = heapInfo->pageHeapMode; if (heapBlock->pageHeapMode == PageHeapMode::PageHeapModeBlockEnd) { // TODO: pad the address to close-most to the guard page to increase the chance to hit guard page when overflow // some Mark code need to be updated to support this // heapBlock->SetEndAllocAddress(address // + AutoSystemInfo::PageSize - (((AllocSizeMath::Add(sizeCat, sizeof(LargeObjectHeader)) - 1) % AutoSystemInfo::PageSize) / HeapInfo::ObjectGranularity + 1) * HeapInfo::ObjectGranularity); } #if DBG LargeAllocationVerboseTrace(recycler->GetRecyclerFlagsTable(), _u("Allocated new large heap block 0x%p for sizeCat 0x%x\n"), heapBlock, sizeCat); #endif #ifdef ENABLE_JS_ETW #if ENABLE_DEBUG_CONFIG_OPTIONS if (segment->GetPageCount() > recycler->GetRecyclerLargeBlockPageAllocator()->GetMaxAllocPageCount()) { EventWriteJSCRIPT_INTERNAL_RECYCLER_EXTRALARGE_OBJECT_ALLOC(size); } #endif #endif #if ENABLE_PARTIAL_GC recycler->autoHeap.uncollectedNewPageCount += pageCount; #endif RECYCLER_SLOW_CHECK(this->heapInfo->heapBlockCount[HeapBlock::HeapBlockType::LargeBlockType]++); heapBlock->heapInfo = this->heapInfo; Assert(recycler->collectionState != CollectionStateMark); if (!recycler->heapBlockMap.SetHeapBlock(address, pageCount, heapBlock, HeapBlock::HeapBlockType::LargeBlockType, 0)) { recycler->GetRecyclerLargeBlockPageAllocator()->SuspendIdleDecommit(); heapBlock->ReleasePages<true>(recycler); recycler->GetRecyclerLargeBlockPageAllocator()->ResumeIdleDecommit(); LargeHeapBlock::Delete(heapBlock); RECYCLER_SLOW_CHECK(this->heapInfo->heapBlockCount[HeapBlock::HeapBlockType::LargeBlockType]--); return nullptr; } heapBlock->ResetMarks(ResetMarkFlags_None, recycler); if (this->largePageHeapBlockList) { HeapBlockList::Tail(this->largePageHeapBlockList)->SetNextBlock(heapBlock); } else { this->largePageHeapBlockList = heapBlock; } RECYCLER_PERF_COUNTER_ADD(FreeObjectSize, heapBlock->GetPageCount() * AutoSystemInfo::PageSize); char * memBlock = heapBlock->Alloc(sizeCat, attributes); Assert(memBlock != nullptr); if (recycler->ShouldCapturePageHeapAllocStack()) { heapBlock->CapturePageHeapAllocStack(); } return memBlock; }