void Heap::collect(SweepToggle sweepToggle)
{
GCPHASE(Collect);
ASSERT(globalData()->identifierTable == wtfThreadData().currentIdentifierTable());
ASSERT(m_isSafeToCollect);
JAVASCRIPTCORE_GC_BEGIN();
#if ENABLE(GGC)
bool fullGC = sweepToggle == DoSweep;
if (!fullGC)
fullGC = (capacity() > 4 * m_lastFullGCSize);
#else
bool fullGC = true;
#endif
{
GCPHASE(Canonicalize);
canonicalizeCellLivenessData();
}
markRoots(fullGC);
{
GCPHASE(HarvestWeakReferences);
harvestWeakReferences();
m_handleHeap.finalizeWeakHandles();
m_globalData->smallStrings.finalizeSmallStrings();
}
JAVASCRIPTCORE_GC_MARKED();
{
GCPHASE(ResetAllocator);
resetAllocator();
}
if (sweepToggle == DoSweep) {
GCPHASE(Sweeping);
sweep();
shrink();
}
// To avoid pathological GC churn in large heaps, we set the allocation high
// water mark to be proportional to the current size of the heap. The exact
// proportion is a bit arbitrary. A 2X multiplier gives a 1:1 (heap size :
// new bytes allocated) proportion, and seems to work well in benchmarks.
size_t newSize = size();
size_t proportionalBytes = 2 * newSize;
if (fullGC) {
m_lastFullGCSize = newSize;
m_objectSpace.setHighWaterMark(max(proportionalBytes, m_minBytesPerCycle));
}
JAVASCRIPTCORE_GC_END();
(*m_activityCallback)();
}
void Heap::willStartCollection(HeapOperation collectionType)
{
GCPHASE(StartingCollection);
if (shouldDoFullCollection(collectionType)) {
m_operationInProgress = FullCollection;
m_slotVisitor.clearMarkStack();
m_shouldDoFullCollection = false;
if (Options::logGC())
dataLog("FullCollection, ");
} else {
m_operationInProgress = EdenCollection;
if (Options::logGC())
dataLog("EdenCollection, ");
}
if (m_operationInProgress == FullCollection) {
m_sizeBeforeLastFullCollect = m_sizeAfterLastCollect + m_bytesAllocatedThisCycle;
m_extraMemorySize = 0;
m_deprecatedExtraMemorySize = 0;
if (m_fullActivityCallback)
m_fullActivityCallback->willCollect();
} else {
ASSERT(m_operationInProgress == EdenCollection);
m_sizeBeforeLastEdenCollect = m_sizeAfterLastCollect + m_bytesAllocatedThisCycle;
}
if (m_edenActivityCallback)
m_edenActivityCallback->willCollect();
}
void Heap::didFinishCollection(double gcStartTime)
{
GCPHASE(FinishingCollection);
double gcEndTime = WTF::monotonicallyIncreasingTime();
if (m_operationInProgress == FullCollection)
m_lastFullGCLength = gcEndTime - gcStartTime;
else
m_lastEdenGCLength = gcEndTime - gcStartTime;
if (Options::recordGCPauseTimes())
HeapStatistics::recordGCPauseTime(gcStartTime, gcEndTime);
if (Options::useZombieMode())
zombifyDeadObjects();
if (Options::objectsAreImmortal())
markDeadObjects();
if (Options::showObjectStatistics())
HeapStatistics::showObjectStatistics(this);
if (Options::logGC() == GCLogging::Verbose)
GCLogging::dumpObjectGraph(this);
RELEASE_ASSERT(m_operationInProgress == EdenCollection || m_operationInProgress == FullCollection);
m_operationInProgress = NoOperation;
JAVASCRIPTCORE_GC_END();
}
void Heap::converge()
{
#if ENABLE(PARALLEL_GC)
GCPHASE(Convergence);
m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain);
#endif
}
void Heap::deleteUnmarkedCompiledCode()
{
GCPHASE(DeleteCodeBlocks);
clearUnmarkedExecutables();
m_codeBlocks.deleteUnmarkedAndUnreferenced(m_operationInProgress);
m_jitStubRoutines.deleteUnmarkedJettisonedStubRoutines();
}
void Heap::stopAllocation()
{
GCPHASE(StopAllocation);
m_objectSpace.stopAllocating();
if (m_operationInProgress == FullCollection)
m_storageSpace.didStartFullCollection();
}
void Heap::markRoots(double gcStartTime, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters)
{
SamplingRegion samplingRegion("Garbage Collection: Marking");
GCPHASE(MarkRoots);
ASSERT(isValidThreadState(m_vm));
#if ENABLE(GGC)
Vector<const JSCell*> rememberedSet(m_slotVisitor.markStack().size());
m_slotVisitor.markStack().fillVector(rememberedSet);
#else
Vector<const JSCell*> rememberedSet;
#endif
#if ENABLE(DFG_JIT)
DFG::clearCodeBlockMarks(*m_vm);
#endif
if (m_operationInProgress == EdenCollection)
m_codeBlocks.clearMarksForEdenCollection(rememberedSet);
else
m_codeBlocks.clearMarksForFullCollection();
// We gather conservative roots before clearing mark bits because conservative
// gathering uses the mark bits to determine whether a reference is valid.
ConservativeRoots conservativeRoots(&m_objectSpace.blocks(), &m_storageSpace);
gatherStackRoots(conservativeRoots, stackOrigin, stackTop, calleeSavedRegisters);
gatherJSStackRoots(conservativeRoots);
gatherScratchBufferRoots(conservativeRoots);
clearLivenessData();
m_sharedData.didStartMarking();
m_slotVisitor.didStartMarking();
HeapRootVisitor heapRootVisitor(m_slotVisitor);
{
ParallelModeEnabler enabler(m_slotVisitor);
visitExternalRememberedSet();
visitSmallStrings();
visitConservativeRoots(conservativeRoots);
visitProtectedObjects(heapRootVisitor);
visitArgumentBuffers(heapRootVisitor);
visitException(heapRootVisitor);
visitStrongHandles(heapRootVisitor);
visitHandleStack(heapRootVisitor);
traceCodeBlocksAndJITStubRoutines();
converge();
}
// Weak references must be marked last because their liveness depends on
// the liveness of the rest of the object graph.
visitWeakHandles(heapRootVisitor);
clearRememberedSet(rememberedSet);
m_sharedData.didFinishMarking();
updateObjectCounts(gcStartTime);
resetVisitors();
}
void Heap::removeDeadCompilerWorklistEntries()
{
#if ENABLE(DFG_JIT)
GCPHASE(FinalizeDFGWorklists);
for (auto worklist : m_suspendedCompilerWorklists)
worklist->removeDeadPlans(*m_vm);
#endif
}
void Heap::pruneStaleEntriesFromWeakGCMaps()
{
GCPHASE(PruningStaleEntriesFromWeakGCMaps);
if (m_operationInProgress != FullCollection)
return;
for (auto& pruneCallback : m_weakGCMaps.values())
pruneCallback();
}
void Heap::gatherJSStackRoots(ConservativeRoots& roots)
{
#if !ENABLE(JIT)
GCPHASE(GatherJSStackRoots);
stack().gatherConservativeRoots(roots, m_jitStubRoutines, m_codeBlocks);
#else
UNUSED_PARAM(roots);
#endif
}
void Heap::resumeCompilerThreads()
{
#if ENABLE(DFG_JIT)
GCPHASE(ResumeCompilerThreads);
for (auto worklist : m_suspendedCompilerWorklists)
worklist->resumeAllThreads();
m_suspendedCompilerWorklists.clear();
#endif
}
void Heap::flushWriteBarrierBuffer()
{
GCPHASE(FlushWriteBarrierBuffer);
if (m_operationInProgress == EdenCollection) {
m_writeBarrierBuffer.flush(*this);
return;
}
m_writeBarrierBuffer.reset();
}
void Heap::gatherScratchBufferRoots(ConservativeRoots& roots)
{
#if ENABLE(DFG_JIT)
GCPHASE(GatherScratchBufferRoots);
m_vm->gatherConservativeRoots(roots);
#else
UNUSED_PARAM(roots);
#endif
}
void Heap::visitConservativeRoots(ConservativeRoots& roots)
{
GCPHASE(VisitConservativeRoots);
m_slotVisitor.append(roots);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Conservative Roots:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::clearRememberedSet(Vector<const JSCell*>& rememberedSet)
{
#if ENABLE(GGC)
GCPHASE(ClearRememberedSet);
for (auto* cell : rememberedSet)
const_cast<JSCell*>(cell)->setRemembered(false);
#else
UNUSED_PARAM(rememberedSet);
#endif
}
void Heap::visitHandleStack(HeapRootVisitor& visitor)
{
GCPHASE(VisitHandleStack);
m_handleStack.visit(visitor);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Handle Stack:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::visitStrongHandles(HeapRootVisitor& visitor)
{
GCPHASE(VisitStrongHandles);
m_handleSet.visitStrongHandles(visitor);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Strong Handles:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::notifyIncrementalSweeper()
{
GCPHASE(NotifyIncrementalSweeper);
if (m_operationInProgress == FullCollection) {
if (!m_logicallyEmptyWeakBlocks.isEmpty())
m_indexOfNextLogicallyEmptyWeakBlockToSweep = 0;
}
m_sweeper->startSweeping();
}
void Heap::traceCodeBlocksAndJITStubRoutines()
{
GCPHASE(TraceCodeBlocksAndJITStubRoutines);
m_codeBlocks.traceMarked(m_slotVisitor);
m_jitStubRoutines.traceMarkedStubRoutines(m_slotVisitor);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Code Blocks and JIT Stub Routines:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::visitSmallStrings()
{
GCPHASE(VisitSmallStrings);
if (!m_vm->smallStrings.needsToBeVisited(m_operationInProgress))
return;
m_vm->smallStrings.visitStrongReferences(m_slotVisitor);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Small strings:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::visitProtectedObjects(HeapRootVisitor& heapRootVisitor)
{
GCPHASE(VisitProtectedObjects);
for (auto& pair : m_protectedValues)
heapRootVisitor.visit(&pair.key);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Protected Objects:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::suspendCompilerThreads()
{
#if ENABLE(DFG_JIT)
GCPHASE(SuspendCompilerThreads);
ASSERT(m_suspendedCompilerWorklists.isEmpty());
for (unsigned i = DFG::numberOfWorklists(); i--;) {
if (DFG::Worklist* worklist = DFG::worklistForIndexOrNull(i)) {
m_suspendedCompilerWorklists.append(worklist);
worklist->suspendAllThreads();
}
}
#endif
}
void Heap::visitArgumentBuffers(HeapRootVisitor& visitor)
{
GCPHASE(MarkingArgumentBuffers);
if (!m_markListSet || !m_markListSet->size())
return;
MarkedArgumentBuffer::markLists(visitor, *m_markListSet);
if (Options::logGC() == GCLogging::Verbose)
dataLog("Argument Buffers:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::visitException(HeapRootVisitor& visitor)
{
GCPHASE(MarkingException);
if (!m_vm->exception() && !m_vm->lastException())
return;
visitor.visit(m_vm->addressOfException());
visitor.visit(m_vm->addressOfLastException());
if (Options::logGC() == GCLogging::Verbose)
dataLog("Exceptions:\n", m_slotVisitor);
m_slotVisitor.donateAndDrain();
}
void Heap::snapshotMarkedSpace()
{
GCPHASE(SnapshotMarkedSpace);
if (m_operationInProgress == EdenCollection) {
m_blockSnapshot.appendVector(m_objectSpace.blocksWithNewObjects());
// Sort and deduplicate the block snapshot since we might be appending to an unfinished work list.
std::sort(m_blockSnapshot.begin(), m_blockSnapshot.end());
m_blockSnapshot.shrink(std::unique(m_blockSnapshot.begin(), m_blockSnapshot.end()) - m_blockSnapshot.begin());
} else {
m_blockSnapshot.resizeToFit(m_objectSpace.blocks().set().size());
MarkedBlockSnapshotFunctor functor(m_blockSnapshot);
m_objectSpace.forEachBlock(functor);
}
}
void Heap::clearUnmarkedExecutables()
{
GCPHASE(ClearUnmarkedExecutables);
for (unsigned i = m_executables.size(); i--;) {
ExecutableBase* current = m_executables[i];
if (isMarked(current))
continue;
// We do this because executable memory is limited on some platforms and because
// CodeBlock requires eager finalization.
ExecutableBase::clearCodeVirtual(current);
std::swap(m_executables[i], m_executables.last());
m_executables.removeLast();
}
}
void Heap::visitWeakHandles(HeapRootVisitor& visitor)
{
GCPHASE(VisitingLiveWeakHandles);
while (true) {
m_objectSpace.visitWeakSets(visitor);
harvestWeakReferences();
visitCompilerWorklistWeakReferences();
if (m_slotVisitor.isEmpty())
break;
if (Options::logGC() == GCLogging::Verbose)
dataLog("Live Weak Handles:\n", m_slotVisitor);
{
ParallelModeEnabler enabler(m_slotVisitor);
m_slotVisitor.donateAndDrain();
#if ENABLE(PARALLEL_GC)
m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain);
#endif
}
}
}
void Heap::copyBackingStores()
{
GCPHASE(CopyBackingStores);
if (m_operationInProgress == EdenCollection)
m_storageSpace.startedCopying<EdenCollection>();
else {
ASSERT(m_operationInProgress == FullCollection);
m_storageSpace.startedCopying<FullCollection>();
}
if (m_storageSpace.shouldDoCopyPhase()) {
m_sharedData.didStartCopying();
m_copyVisitor.startCopying();
m_copyVisitor.copyFromShared();
m_copyVisitor.doneCopying();
// We need to wait for everybody to finish and return their CopiedBlocks
// before signaling that the phase is complete.
m_storageSpace.doneCopying();
m_sharedData.didFinishCopying();
} else
m_storageSpace.doneCopying();
}
void Heap::updateAllocationLimits()
{
GCPHASE(UpdateAllocationLimits);
size_t currentHeapSize = sizeAfterCollect();
if (Options::gcMaxHeapSize() && currentHeapSize > Options::gcMaxHeapSize())
HeapStatistics::exitWithFailure();
if (m_operationInProgress == FullCollection) {
// To avoid pathological GC churn in very small and very large heaps, we set
// the new allocation limit based on the current size of the heap, with a
// fixed minimum.
m_maxHeapSize = max(minHeapSize(m_heapType, m_ramSize), proportionalHeapSize(currentHeapSize, m_ramSize));
m_maxEdenSize = m_maxHeapSize - currentHeapSize;
m_sizeAfterLastFullCollect = currentHeapSize;
m_bytesAbandonedSinceLastFullCollect = 0;
} else {
ASSERT(currentHeapSize >= m_sizeAfterLastCollect);
m_maxEdenSize = m_maxHeapSize - currentHeapSize;
m_sizeAfterLastEdenCollect = currentHeapSize;
double edenToOldGenerationRatio = (double)m_maxEdenSize / (double)m_maxHeapSize;
double minEdenToOldGenerationRatio = 1.0 / 3.0;
if (edenToOldGenerationRatio < minEdenToOldGenerationRatio)
m_shouldDoFullCollection = true;
m_maxHeapSize += currentHeapSize - m_sizeAfterLastCollect;
m_maxEdenSize = m_maxHeapSize - currentHeapSize;
if (m_fullActivityCallback) {
ASSERT(currentHeapSize >= m_sizeAfterLastFullCollect);
m_fullActivityCallback->didAllocate(currentHeapSize - m_sizeAfterLastFullCollect);
}
}
m_sizeAfterLastCollect = currentHeapSize;
m_bytesAllocatedThisCycle = 0;
if (Options::logGC())
dataLog(currentHeapSize / 1024, " kb, ");
}
void Heap::markRoots(bool fullGC)
{
COND_GCPHASE(fullGC, MarkFullRoots, MarkYoungRoots);
UNUSED_PARAM(fullGC);
ASSERT(isValidThreadState(m_globalData));
if (m_operationInProgress != NoOperation)
CRASH();
m_operationInProgress = Collection;
void* dummy;
// We gather conservative roots before clearing mark bits because conservative
// gathering uses the mark bits to determine whether a reference is valid.
ConservativeRoots machineThreadRoots(&m_objectSpace.blocks());
{
GCPHASE(GatherConservativeRoots);
m_machineThreads.gatherConservativeRoots(machineThreadRoots, &dummy);
}
ConservativeRoots registerFileRoots(&m_objectSpace.blocks());
m_jettisonedCodeBlocks.clearMarks();
{
GCPHASE(GatherRegisterFileRoots);
registerFile().gatherConservativeRoots(registerFileRoots, m_jettisonedCodeBlocks);
}
m_jettisonedCodeBlocks.deleteUnmarkedCodeBlocks();
#if ENABLE(GGC)
MarkedBlock::DirtyCellVector dirtyCells;
if (!fullGC) {
GCPHASE(GatheringDirtyCells);
m_objectSpace.gatherDirtyCells(dirtyCells);
} else
#endif
{
GCPHASE(clearMarks);
clearMarks();
}
SlotVisitor& visitor = m_slotVisitor;
HeapRootVisitor heapRootVisitor(visitor);
#if ENABLE(GGC)
{
size_t dirtyCellCount = dirtyCells.size();
GCPHASE(VisitDirtyCells);
GCCOUNTER(DirtyCellCount, dirtyCellCount);
for (size_t i = 0; i < dirtyCellCount; i++) {
heapRootVisitor.visitChildren(dirtyCells[i]);
visitor.drain();
}
}
#endif
if (m_globalData->codeBlocksBeingCompiled.size()) {
GCPHASE(VisitActiveCodeBlock);
for (size_t i = 0; i < m_globalData->codeBlocksBeingCompiled.size(); i++)
m_globalData->codeBlocksBeingCompiled[i]->visitAggregate(visitor);
}
{
GCPHASE(VisitMachineRoots);
visitor.append(machineThreadRoots);
visitor.drain();
}
{
GCPHASE(VisitRegisterFileRoots);
visitor.append(registerFileRoots);
visitor.drain();
}
{
GCPHASE(VisitProtectedObjects);
markProtectedObjects(heapRootVisitor);
visitor.drain();
}
{
GCPHASE(VisitTempSortVectors);
markTempSortVectors(heapRootVisitor);
visitor.drain();
}
{
GCPHASE(MarkingArgumentBuffers);
if (m_markListSet && m_markListSet->size()) {
MarkedArgumentBuffer::markLists(heapRootVisitor, *m_markListSet);
visitor.drain();
}
}
if (m_globalData->exception) {
GCPHASE(MarkingException);
heapRootVisitor.visit(&m_globalData->exception);
visitor.drain();
}
{
GCPHASE(VisitStrongHandles);
m_handleHeap.visitStrongHandles(heapRootVisitor);
visitor.drain();
}
{
GCPHASE(HandleStack);
m_handleStack.visit(heapRootVisitor);
visitor.drain();
}
{
GCPHASE(TraceCodeBlocks);
m_jettisonedCodeBlocks.traceCodeBlocks(visitor);
visitor.drain();
}
// Weak handles must be marked last, because their owners use the set of
// opaque roots to determine reachability.
{
GCPHASE(VisitingWeakHandles);
int lastOpaqueRootCount;
do {
lastOpaqueRootCount = visitor.opaqueRootCount();
m_handleHeap.visitWeakHandles(heapRootVisitor);
visitor.drain();
// If the set of opaque roots has grown, more weak handles may have become reachable.
} while (lastOpaqueRootCount != visitor.opaqueRootCount());
}
GCCOUNTER(VisitedValueCount, visitor.visitCount());
visitor.reset();
m_operationInProgress = NoOperation;
}
