void GCCopyPhase()
{
mainThreadPhase = MTP_GCPHASECOMPACT;
for(std::vector<LocalMemSpace*>::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++)
{
LocalMemSpace *lSpace = *i;
uintptr_t highest = lSpace->wordNo(lSpace->top);
for (unsigned i = 0; i < NSTARTS; i++)
lSpace->start[i] = highest;
lSpace->start_index = NSTARTS - 1;
lSpace->spaceOwner = 0;
// Reset the allocation pointers. This puts garbage (and real data) below them.
// At the end of the compaction the allocation pointer will point below the
// lowest real data.
lSpace->upperAllocPtr = lSpace->top;
}
// Copy the mutable data into a lower area if possible.
if (gpTaskFarm->ThreadCount() == 0)
copyAllData(globalTask, 0, 0);
else
{
// We start as many tasks as we have threads. If the amount of work to
// be done is very small one thread could process more than one task.
// Have to be careful because we use the task ID to decide which space
// to scan.
for (unsigned j = 0; j < gpTaskFarm->ThreadCount(); j++)
gpTaskFarm->AddWorkOrRunNow(©AllData, 0, 0);
}
gpTaskFarm->WaitForCompletion();
}
// Report heap sizes and occupancy before and after GC
void MemMgr::ReportHeapSizes(const char *phase)
{
POLYUNSIGNED alloc = 0, nonAlloc = 0, inAlloc = 0, inNonAlloc = 0;
for (unsigned i = 0; i < nlSpaces; i++)
{
LocalMemSpace *sp = lSpaces[i];
if (sp->allocationSpace)
{
alloc += sp->spaceSize();
inAlloc += sp->allocatedSpace();
}
else
{
nonAlloc += sp->spaceSize();
inNonAlloc += sp->allocatedSpace();
}
}
Log("Heap: %s Major heap used ", phase);
LogSize(inNonAlloc); Log(" of ");
LogSize(nonAlloc);
Log(" (%1.0f%%). Alloc space used ", (float)inNonAlloc / (float)nonAlloc * 100.0F);
LogSize(inAlloc); Log(" of ");
LogSize(alloc);
Log(" (%1.0f%%). Total space ", (float)inAlloc / (float)alloc * 100.0F);
LogSize(spaceForHeap);
Log(" %1.0f%% full.\n", (float)(inAlloc + inNonAlloc) / (float)spaceForHeap * 100.0F);
}
bool GetSharing::TestForScan(PolyWord *pt)
{
PolyObject *obj;
// This may be a forwarding pointer left over from a minor GC that did
// not complete or it may be a sharing chain pointer that we've set up.
while (1)
{
PolyWord p = *pt;
ASSERT(p.IsDataPtr());
obj = p.AsObjPtr();
PolyWord *lengthWord = ((PolyWord*)obj) - 1;
LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
if (space == 0)
return false; // Ignore it if it points to a permanent area
if (space->bitmap.TestBit(space->wordNo(lengthWord)))
return false;
// Wasn't marked - must be a forwarding pointer.
if (obj->ContainsForwardingPtr())
{
obj = obj->GetForwardingPtr();
*pt = obj;
}
else break;
}
ASSERT(obj->ContainsNormalLengthWord());
totalVisited += 1;
totalSize += obj->Length() + 1;
return true;
}
void GetSharing::MarkAsScanning(PolyObject *obj)
{
ASSERT(obj->ContainsNormalLengthWord());
PolyWord *lengthWord = ((PolyWord*)obj) - 1;
LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
ASSERT(! space->bitmap.TestBit(space->wordNo(lengthWord)));
space->bitmap.SetBit(space->wordNo(lengthWord));
}
// Remove local areas that are now empty after a GC.
// It isn't clear if we always want to do this.
void MemMgr::RemoveEmptyLocals()
{
for (unsigned s = nlSpaces; s > 0; s--)
{
LocalMemSpace *space = lSpaces[s-1];
if (space->allocatedSpace() == 0)
DeleteLocalSpace(space);
}
}
// If we have saved the state rather than exported a function we turn the exported
// spaces into permanent ones, removing existing permanent spaces at the same or
// lower level.
bool MemMgr::PromoteExportSpaces(unsigned hierarchy)
{
// Create a new table big enough to hold all the permanent and export spaces
PermanentMemSpace **pTable =
(PermanentMemSpace **)calloc(npSpaces+neSpaces, sizeof(PermanentMemSpace *));
if (pTable == 0) return false;
unsigned newSpaces = 0;
// Save permanent spaces at a lower hierarchy. Others are converted into
// local spaces. Most or all items will have been copied from these spaces
// into an export space but there could be items reachable only from the stack.
for (unsigned i = 0; i < npSpaces; i++)
{
PermanentMemSpace *pSpace = pSpaces[i];
if (pSpace->hierarchy < hierarchy)
pTable[newSpaces++] = pSpace;
else
{
try {
// Turn this into a local space.
// Remove this from the tree - AddLocalSpace will make an entry for the local version.
RemoveTree(pSpace);
LocalMemSpace *space = new LocalMemSpace;
space->top = space->fullGCLowerLimit = pSpace->top;
space->bottom = space->upperAllocPtr = space->lowerAllocPtr = pSpace->bottom;
space->isMutable = pSpace->isMutable;
space->isOwnSpace = true;
if (! space->bitmap.Create(space->top-space->bottom) || ! AddLocalSpace(space))
return false;
currentHeapSize += space->spaceSize();
globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord));
}
catch (std::bad_alloc a) {
return false;
}
}
}
// Save newly exported spaces.
for (unsigned j = 0; j < neSpaces; j++)
{
PermanentMemSpace *space = eSpaces[j];
space->hierarchy = hierarchy; // Set the hierarchy of the new spaces.
space->spaceType = ST_PERMANENT;
// Put a dummy object to fill up the unused space.
if (space->topPointer != space->top)
FillUnusedSpace(space->topPointer, space->top - space->topPointer);
// Put in a dummy object to fill the rest of the space.
pTable[newSpaces++] = space;
}
neSpaces = 0;
npSpaces = newSpaces;
free(pSpaces);
pSpaces = pTable;
return true;
}
// Create a local space for initial allocation.
LocalMemSpace *MemMgr::CreateAllocationSpace(POLYUNSIGNED size)
{
LocalMemSpace *result = NewLocalSpace(size, true);
if (result)
{
result->allocationSpace = true;
currentAllocSpace += result->spaceSize();
globalStats.incSize(PSS_ALLOCATION, result->spaceSize()*sizeof(PolyWord));
globalStats.incSize(PSS_ALLOCATION_FREE, result->freeSpace()*sizeof(PolyWord));
}
return result;
}
// Allocate an area of the heap of at least minWords and at most maxWords.
// This is used both when allocating single objects (when minWords and maxWords
// are the same) and when allocating heap segments. If there is insufficient
// space to satisfy the minimum it will return 0.
PolyWord *MemMgr::AllocHeapSpace(POLYUNSIGNED minWords, POLYUNSIGNED &maxWords, bool doAllocation)
{
PLocker locker(&allocLock);
// We try to distribute the allocations between the memory spaces
// so that at the next GC we don't have all the most recent cells in
// one space. The most recent cells will be more likely to survive a
// GC so distibuting them improves the load balance for a multi-thread GC.
nextAllocator++;
if (nextAllocator > gMem.nlSpaces) nextAllocator = 0;
for (unsigned j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *space = gMem.lSpaces[(j + nextAllocator) % gMem.nlSpaces];
if (space->allocationSpace)
{
POLYUNSIGNED available = space->freeSpace();
if (available > 0 && available >= minWords)
{
// Reduce the maximum value if we had less than that.
if (available < maxWords)
maxWords = available;
PolyWord *result = space->lowerAllocPtr; // Return the address.
if (doAllocation)
space->lowerAllocPtr += maxWords; // Allocate it.
return result;
}
}
}
// There isn't space in the existing areas - can we create a new area?
// The reason we don't have enough space could simply be that we want to
// allocate an object larger than the default space size. Try deleting
// some other spaces to bring currentAllocSpace below spaceBeforeMinorGC - minWords.
if (minWords > defaultSpaceSize && minWords < spaceBeforeMinorGC)
RemoveExcessAllocation(spaceBeforeMinorGC - minWords);
if (currentAllocSpace < spaceBeforeMinorGC && minWords < spaceBeforeMinorGC - currentAllocSpace)
{
POLYUNSIGNED spaceSize = defaultSpaceSize;
if (minWords > spaceSize) spaceSize = minWords; // If we really want a large space.
LocalMemSpace *space = CreateAllocationSpace(spaceSize);
if (space == 0) return 0; // Can't allocate it
// Allocate our space in this new area.
POLYUNSIGNED available = space->freeSpace();
ASSERT(available >= minWords);
if (available < maxWords)
maxWords = available;
PolyWord *result = space->lowerAllocPtr; // Return the address.
if (doAllocation)
space->lowerAllocPtr += maxWords; // Allocate it.
return result;
}
return 0; // There isn't space even for the minimum.
}
// Return number of words free in all allocation spaces.
POLYUNSIGNED MemMgr::GetFreeAllocSpace()
{
POLYUNSIGNED freeSpace = 0;
PLocker lock(&allocLock);
for (unsigned j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *space = gMem.lSpaces[j];
if (space->allocationSpace)
freeSpace += space->freeSpace();
}
return freeSpace;
}
// Set the forwarding so that references to objToSet will be forwarded to
// objToShare. objToSet will be garbage.
void shareWith(PolyObject *objToSet, PolyObject *objToShare)
{
// We need to remove the bit from this so that we know it's not
// a share chain.
PolyWord *lengthWord = ((PolyWord*)objToSet) - 1;
LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
ASSERT(space);
PLocker locker(&space->bitmapLock);
ASSERT(space->bitmap.TestBit(space->wordNo(lengthWord)));
space->bitmap.ClearBit(space->wordNo(lengthWord));
// Actually do the forwarding
objToSet->SetForwardingPtr(objToShare);
}
// Before we import a hierarchical saved state we need to turn any previously imported
// spaces into local spaces.
bool MemMgr::DemoteImportSpaces()
{
// Create a new permanent space table.
PermanentMemSpace **table =
(PermanentMemSpace **)calloc(npSpaces, sizeof(PermanentMemSpace *));
if (table == NULL) return false;
unsigned newSpaces = 0;
for (unsigned i = 0; i < npSpaces; i++)
{
PermanentMemSpace *pSpace = pSpaces[i];
if (pSpace->hierarchy == 0) // Leave truly permanent spaces
table[newSpaces++] = pSpace;
else
{
try {
// Turn this into a local space.
// Remove this from the tree - AddLocalSpace will make an entry for the local version.
RemoveTree(pSpace);
LocalMemSpace *space = new LocalMemSpace;
space->top = pSpace->top;
// Space is allocated in local areas from the top down. This area is full and
// all data is in the old generation. The area can be recovered by a full GC.
space->bottom = space->upperAllocPtr = space->lowerAllocPtr =
space->fullGCLowerLimit = pSpace->bottom;
space->isMutable = pSpace->isMutable;
space->isOwnSpace = true;
if (! space->bitmap.Create(space->top-space->bottom) || ! AddLocalSpace(space))
{
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: Unable to convert saved state space %p into local space\n", pSpace);
return false;
}
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: Converted saved state space %p into local %smutable space %p\n",
pSpace, pSpace->isMutable ? "im": "", space);
currentHeapSize += space->spaceSize();
globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord));
}
catch (std::bad_alloc a) {
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: Unable to convert saved state space %p into local space (\"new\" failed)\n", pSpace);
return false;
}
}
}
npSpaces = newSpaces;
free(pSpaces);
pSpaces = table;
return true;
}
objectState getObjectState(PolyObject *p)
{
PolyWord *lengthWord = ((PolyWord*)p) - 1;
LocalMemSpace *space = gMem.LocalSpaceForAddress(lengthWord);
if (space == 0)
return REALOBJECT; // May be the address of a permanent or something else.
PLocker locker(&space->bitmapLock);
if (!p->ContainsForwardingPtr())
return REALOBJECT;
if (space->bitmap.TestBit(space->wordNo(lengthWord)))
return CHAINED;
else return FORWARDED;
}
void GCSharingPhase(void)
{
mainThreadPhase = MTP_GCPHASESHARING;
GetSharing sharer;
for (std::vector<LocalMemSpace*>::iterator i = gMem.lSpaces.begin(); i < gMem.lSpaces.end(); i++)
{
LocalMemSpace *lSpace = *i;
lSpace->bitmap.ClearBits(0, lSpace->spaceSize());
}
// Scan the code areas to share any constants. We don't share the code
// cells themselves.
for (std::vector<CodeSpace *>::iterator i = gMem.cSpaces.begin(); i < gMem.cSpaces.end(); i++)
{
CodeSpace *space = *i;
sharer.ScanAddressesInRegion(space->bottom, space->top);
}
if (debugOptions & DEBUG_GC)
Log("GC: Share: After scanning code: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n",
sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded);
// Process the permanent mutable areas and the code areas
for (std::vector<PermanentMemSpace*>::iterator i = gMem.pSpaces.begin(); i < gMem.pSpaces.end(); i++)
{
PermanentMemSpace *space = *i;
if (space->isMutable && ! space->byteOnly)
sharer.ScanAddressesInRegion(space->bottom, space->top);
}
if (debugOptions & DEBUG_GC)
Log("GC: Share: After scanning permanent: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n",
sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded);
// Process the RTS roots.
GCModules(&sharer);
if (debugOptions & DEBUG_GC)
Log("GC: Share: After scanning other roots: Total %" POLYUFMT " (%" POLYUFMT " words) byte %" POLYUFMT " word %" POLYUFMT ".\n",
sharer.totalVisited, sharer.totalSize, sharer.byteAdded, sharer.wordAdded);
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Table");
// Sort and merge the data.
sharer.SortData();
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Sort");
}
void GCMarkPhase(void)
{
mainThreadPhase = MTP_GCPHASEMARK;
// Clear the mark counters and set the rescan limits.
for(unsigned k = 0; k < gMem.nlSpaces; k++)
{
LocalMemSpace *lSpace = gMem.lSpaces[k];
lSpace->i_marked = lSpace->m_marked = 0;
lSpace->fullGCRescanStart = lSpace->top;
lSpace->fullGCRescanEnd = lSpace->bottom;
}
MTGCProcessMarkPointers::MarkRoots();
gpTaskFarm->WaitForCompletion();
// Do we have to rescan because the mark stack overflowed?
bool rescan;
do {
rescan = MTGCProcessMarkPointers::RescanForStackOverflow();
gpTaskFarm->WaitForCompletion();
} while(rescan);
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Mark");
// Turn the marks into bitmap entries.
for (unsigned i = 0; i < gMem.nlSpaces; i++)
gpTaskFarm->AddWorkOrRunNow(&CreateBitmapsTask, gMem.lSpaces[i], 0);
gpTaskFarm->WaitForCompletion(); // Wait for completion of the bitmaps
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Bitmap");
POLYUNSIGNED totalLive = 0;
for(unsigned l = 0; l < gMem.nlSpaces; l++)
{
LocalMemSpace *lSpace = gMem.lSpaces[l];
if (! lSpace->isMutable) ASSERT(lSpace->m_marked == 0);
totalLive += lSpace->m_marked + lSpace->i_marked;
if (debugOptions & DEBUG_GC)
Log("GC: Mark: %s space %p: %" POLYUFMT " immutable words marked, %" POLYUFMT " mutable words marked\n",
lSpace->spaceTypeString(), lSpace,
lSpace->i_marked, lSpace->m_marked);
}
if (debugOptions & DEBUG_GC)
Log("GC: Mark: Total live data %" POLYUFMT " words\n", totalLive);
}
// Create and initialise a new local space and add it to the table.
LocalMemSpace* MemMgr::NewLocalSpace(POLYUNSIGNED size, bool mut)
{
try {
LocalMemSpace *space = new LocalMemSpace;
// Before trying to allocate the heap temporarily allocate the
// reserved space. This ensures that this much space will always
// be available for C stacks and the C++ heap.
void *reservation = 0;
size_t rSpace = reservedSpace*sizeof(PolyWord);
if (reservedSpace != 0) {
reservation = osMemoryManager->Allocate(rSpace, PERMISSION_READ);
if (reservation == 0) {
// Insufficient space for the reservation. Can't allocate this local space.
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: New local %smutable space: insufficient reservation space\n", mut ? "": "im");
delete space;
return 0;
}
}
bool success = space->InitSpace(size, mut) && AddLocalSpace(space);
if (reservation != 0) osMemoryManager->Free(reservation, rSpace);
if (success)
{
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: New local %smutable space %p, size=%luk words, bottom=%p, top=%p\n", mut ? "": "im",
space, space->spaceSize()/1024, space->bottom, space->top);
currentHeapSize += space->spaceSize();
globalStats.setSize(PSS_TOTAL_HEAP, currentHeapSize * sizeof(PolyWord));
return space;
}
// If something went wrong.
delete space;
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: New local %smutable space: insufficient space\n", mut ? "": "im");
return 0;
}
catch (std::bad_alloc a) {
if (debugOptions & DEBUG_MEMMGR)
Log("MMGR: New local %smutable space: \"new\" failed\n", mut ? "": "im");
return 0;
}
}
// Adjust the allocation area by removing free areas so that the total
// size of the allocation area is less than the required value. This
// is used after the quick GC and also if we need to allocate a large
// object.
void MemMgr::RemoveExcessAllocation(POLYUNSIGNED words)
{
// First remove any non-standard allocation areas.
unsigned i;
for (i = nlSpaces; i > 0; i--)
{
LocalMemSpace *space = lSpaces[i-1];
if (space->allocationSpace && space->allocatedSpace() == 0 &&
space->spaceSize() != defaultSpaceSize)
DeleteLocalSpace(space);
}
for (i = nlSpaces; currentAllocSpace > words && i > 0; i--)
{
LocalMemSpace *space = lSpaces[i-1];
if (space->allocationSpace && space->allocatedSpace() == 0)
DeleteLocalSpace(space);
}
}
// This deals with weak references within the run-time system.
void MTGCCheckWeakRef::ScanRuntimeAddress(PolyObject **pt, RtsStrength weak)
{
/* If the object has not been marked and this is only a weak reference */
/* then the pointer is set to zero. This allows streams or windows */
/* to be closed if there is no other reference to them. */
PolyObject *val = *pt;
PolyWord w = val;
if (weak == STRENGTH_STRONG)
return;
LocalMemSpace *space = gMem.LocalSpaceForAddress(w.AsStackAddr());
if (space == 0)
return; // Not in local area
// If it hasn't been marked set it to zero.
if (! space->bitmap.TestBit(space->wordNo(w.AsStackAddr())))
*pt = 0;
}
// Deal with weak objects
void MTGCCheckWeakRef::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED L)
{
if (! OBJ_IS_WEAKREF_OBJECT(L)) return;
ASSERT(OBJ_IS_MUTABLE_OBJECT(L)); // Should be a mutable.
ASSERT(OBJ_IS_WORD_OBJECT(L)); // Should be a plain object.
// See if any of the SOME objects contain unreferenced refs.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(L);
PolyWord *baseAddr = (PolyWord*)obj;
for (POLYUNSIGNED i = 0; i < length; i++)
{
PolyWord someAddr = baseAddr[i];
if (someAddr.IsDataPtr())
{
LocalMemSpace *someSpace = gMem.LocalSpaceForAddress(someAddr.AsAddress());
if (someSpace != 0)
{
PolyObject *someObj = someAddr.AsObjPtr();
// If this is a weak object the SOME value may refer to an unreferenced
// ref. If so we have to set this entry to NONE. For safety we also
// set the contents of the SOME to TAGGED(0).
ASSERT(someObj->Length() == 1 && someObj->IsWordObject()); // Should be a SOME node.
PolyWord refAddress = someObj->Get(0);
LocalMemSpace *space = gMem.LocalSpaceForAddress(refAddress.AsAddress());
if (space != 0)
// If the ref is permanent it's always there.
{
POLYUNSIGNED new_bitno = space->wordNo(refAddress.AsStackAddr());
if (! space->bitmap.TestBit(new_bitno))
{
// It wasn't marked so it's otherwise unreferenced.
baseAddr[i] = TAGGED(0); // Set it to NONE.
someObj->Set(0, TAGGED(0)); // For safety.
convertedWeak = true;
}
}
}
}
}
}
static void CreateBitmapsTask(GCTaskId *, void *arg1, void *arg2)
{
LocalMemSpace *lSpace = (LocalMemSpace *)arg1;
lSpace->bitmap.ClearBits(0, lSpace->spaceSize());
SetBitmaps(lSpace, lSpace->bottom, lSpace->top);
}
/*
How the garbage collector works.
The GC has two phases. The minor (quick) GC is a copying collector that
copies data from the allocation area into the mutable and immutable area.
The major collector is started when either the mutable or the immutable
area is full. The major collector uses a mark/sweep scheme.
The GC has three phases:
1. Mark phase.
Working from the roots; which are the the permanent mutable segments and
the RTS roots (e.g. thread stacks), mark all reachable cells.
Marking involves setting bits in the bitmap for reachable words.
2. Compact phase.
Marked objects are copied to try to compact, upwards, the heap segments. When
an object is moved the length word of the object in the old location is set as
a tombstone that points to its new location. In particular this means that we
cannot reuse the space where an object previously was during the compaction phase.
Immutable objects are moved into immutable segments. When an object is moved
to a new location the bits are set in the bitmap as though the object had been
marked at that location.
3. Update phase.
The roots and objects marked during the first two phases are scanned and any
addresses for moved objects are updated. The lowest address used in the area
then becomes the base of the area for future allocations.
There is a sharing phase which may be performed before the mark phase. This
merges immutable cells with the same contents with the aim of reducing the
size of the live data. It is expensive so is not performed by default.
Updated DCJM 12/06/12
*/
static bool doGC(const POLYUNSIGNED wordsRequiredToAllocate)
{
unsigned j;
gHeapSizeParameters.RecordAtStartOfMajorGC();
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeStart);
globalStats.incCount(PSC_GC_FULLGC);
// Remove any empty spaces. There will not normally be any except
// if we have triggered a full GC as a result of detecting paging in the
// minor GC but in that case we want to try to stop the system writing
// out areas that are now empty.
gMem.RemoveEmptyLocals();
if (debugOptions & DEBUG_GC)
Log("GC: Full GC, %lu words required %u spaces\n", wordsRequiredToAllocate, gMem.nlSpaces);
if (debugOptions & DEBUG_HEAPSIZE)
gMem.ReportHeapSizes("Full GC (before)");
// Data sharing pass.
if (gHeapSizeParameters.PerformSharingPass())
GCSharingPhase();
/*
* There is a really weird bug somewhere. An extra bit may be set in the bitmap during
* the mark phase. It seems to be related to heavy swapping activity. Duplicating the
* bitmap causes it to occur only in one copy and write-protecting the bitmap apart from
* when it is actually being updated does not result in a seg-fault. So far I've only
* seen it on 64-bit Linux but it may be responsible for other crashes. The work-around
* is to check the number of bits set in the bitmap and repeat the mark phase if it does
* not match.
*/
for (unsigned p = 3; p > 0; p--)
{
for(j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *lSpace = gMem.lSpaces[j];
ASSERT (lSpace->top >= lSpace->upperAllocPtr);
ASSERT (lSpace->upperAllocPtr >= lSpace->lowerAllocPtr);
ASSERT (lSpace->lowerAllocPtr >= lSpace->bottom);
// Set upper and lower limits of weak refs.
lSpace->highestWeak = lSpace->bottom;
lSpace->lowestWeak = lSpace->top;
lSpace->fullGCLowerLimit = lSpace->top;
// Put dummy objects in the unused space. This allows
// us to scan over the whole of the space.
gMem.FillUnusedSpace(lSpace->lowerAllocPtr,
lSpace->upperAllocPtr-lSpace->lowerAllocPtr);
}
// Set limits of weak refs.
for (j = 0; j < gMem.npSpaces; j++)
{
PermanentMemSpace *pSpace = gMem.pSpaces[j];
pSpace->highestWeak = pSpace->bottom;
pSpace->lowestWeak = pSpace->top;
}
/* Mark phase */
GCMarkPhase();
POLYUNSIGNED bitCount = 0, markCount = 0;
for (j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *lSpace = gMem.lSpaces[j];
markCount += lSpace->i_marked + lSpace->m_marked;
bitCount += lSpace->bitmap.CountSetBits(lSpace->spaceSize());
}
if (markCount == bitCount)
break;
else
{
// Report an error. If this happens again we crash.
Log("GC: Count error for space %u - mark count %lu, bitCount %lu\n", j, markCount, bitCount);
if (p == 1)
{
ASSERT(markCount == bitCount);
}
}
}
for(j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *lSpace = gMem.lSpaces[j];
// Reset the allocation pointers. They will be set to the
// limits of the retained data.
lSpace->lowerAllocPtr = lSpace->bottom;
lSpace->upperAllocPtr = lSpace->top;
}
if (debugOptions & DEBUG_GC) Log("GC: Check weak refs\n");
/* Detect unreferenced streams, windows etc. */
GCheckWeakRefs();
// Check that the heap is not overfull. We make sure the marked
// mutable and immutable data is no more than 90% of the
// corresponding areas. This is a very coarse adjustment.
{
POLYUNSIGNED iMarked = 0, mMarked = 0;
POLYUNSIGNED iSpace = 0, mSpace = 0;
for (unsigned i = 0; i < gMem.nlSpaces; i++)
{
LocalMemSpace *lSpace = gMem.lSpaces[i];
iMarked += lSpace->i_marked;
mMarked += lSpace->m_marked;
if (! lSpace->allocationSpace)
{
if (lSpace->isMutable)
mSpace += lSpace->spaceSize();
else
iSpace += lSpace->spaceSize();
}
}
// Add space if necessary and possible.
while (iMarked > iSpace - iSpace/10 && gHeapSizeParameters.AddSpaceBeforeCopyPhase(false) != 0)
iSpace += gMem.DefaultSpaceSize();
while (mMarked > mSpace - mSpace/10 && gHeapSizeParameters.AddSpaceBeforeCopyPhase(true) != 0)
mSpace += gMem.DefaultSpaceSize();
}
/* Compact phase */
GCCopyPhase();
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Copy");
// Update Phase.
if (debugOptions & DEBUG_GC) Log("GC: Update\n");
GCUpdatePhase();
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeIntermediate, "Update");
{
POLYUNSIGNED iUpdated = 0, mUpdated = 0, iMarked = 0, mMarked = 0;
for(j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *lSpace = gMem.lSpaces[j];
iMarked += lSpace->i_marked;
mMarked += lSpace->m_marked;
if (lSpace->isMutable)
mUpdated += lSpace->updated;
else
iUpdated += lSpace->updated;
}
ASSERT(iUpdated+mUpdated == iMarked+mMarked);
}
// Delete empty spaces.
gMem.RemoveEmptyLocals();
if (debugOptions & DEBUG_GC)
{
for(j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *lSpace = gMem.lSpaces[j];
Log("GC: %s space %p %d free in %d words %2.1f%% full\n", lSpace->spaceTypeString(),
lSpace, lSpace->freeSpace(), lSpace->spaceSize(),
((float)lSpace->allocatedSpace()) * 100 / (float)lSpace->spaceSize());
}
}
// Compute values for statistics
globalStats.setSize(PSS_AFTER_LAST_GC, 0);
globalStats.setSize(PSS_AFTER_LAST_FULLGC, 0);
globalStats.setSize(PSS_ALLOCATION, 0);
globalStats.setSize(PSS_ALLOCATION_FREE, 0);
for (j = 0; j < gMem.nlSpaces; j++)
{
LocalMemSpace *space = gMem.lSpaces[j];
POLYUNSIGNED free = space->freeSpace();
globalStats.incSize(PSS_AFTER_LAST_GC, free*sizeof(PolyWord));
globalStats.incSize(PSS_AFTER_LAST_FULLGC, free*sizeof(PolyWord));
if (space->allocationSpace)
{
globalStats.incSize(PSS_ALLOCATION, free*sizeof(PolyWord));
globalStats.incSize(PSS_ALLOCATION_FREE, free*sizeof(PolyWord));
}
#ifdef FILL_UNUSED_MEMORY
memset(space->bottom, 0xaa, (char*)space->upperAllocPtr - (char*)space->bottom);
#endif
if (debugOptions & DEBUG_GC)
Log("GC: %s space %p %d free in %d words %2.1f%% full\n", space->spaceTypeString(),
space, space->freeSpace(), space->spaceSize(),
((float)space->allocatedSpace()) * 100 / (float)space->spaceSize());
}
// End of garbage collection
gHeapSizeParameters.RecordGCTime(HeapSizeParameters::GCTimeEnd);
// Now we've finished we can adjust the heap sizes.
gHeapSizeParameters.AdjustSizeAfterMajorGC(wordsRequiredToAllocate);
gHeapSizeParameters.resetMajorTimingData();
bool haveSpace = gMem.CheckForAllocation(wordsRequiredToAllocate);
// Invariant: the bitmaps are completely clean.
if (debugOptions & DEBUG_GC)
{
if (haveSpace)
Log("GC: Completed successfully\n");
else Log("GC: Completed with insufficient space\n");
}
if (debugOptions & DEBUG_HEAPSIZE)
gMem.ReportHeapSizes("Full GC (after)");
if (profileMode == kProfileLiveData || profileMode == kProfileLiveMutables)
printprofile();
CheckMemory();
return haveSpace; // Completed
}
// Copy objects from the source space into an earlier space or up within the
// current space.
static void copyAllData(GCTaskId *id, void * /*arg1*/, void * /*arg2*/)
{
LocalMemSpace *mutableDest = 0, *immutableDest = 0;
for (std::vector<LocalMemSpace*>::reverse_iterator i = gMem.lSpaces.rbegin(); i != gMem.lSpaces.rend(); i++)
{
LocalMemSpace *src = *i;
if (src->spaceOwner == 0)
{
PLocker lock(©Lock);
if (src->spaceOwner == 0)
src->spaceOwner = id;
else continue;
}
else if (src->spaceOwner != id)
continue;
if (debugOptions & DEBUG_GC_ENHANCED)
Log("GC: Copy: copying area %p (thread %p) %s \n", src, id, src->spaceTypeString());
// We start at fullGCLowerLimit which is the lowest marked object in the heap
// N.B. It's essential that the first set bit at or above this corresponds
// to the length word of a real object.
uintptr_t bitno = src->wordNo(src->fullGCLowerLimit);
// Set the limit to the top so we won't rescan this. That can
// only happen if copying takes a very short time and the same
// thread runs multiple tasks.
src->fullGCLowerLimit = src->top;
// src->highest is the bit position that corresponds to the top of
// generation we're copying.
uintptr_t highest = src->wordNo(src->top);
for (;;)
{
if (bitno >= highest) break;
/* SPF version; Invariant: 0 < highest - bitno */
bitno += src->bitmap.CountZeroBits(bitno, highest - bitno);
if (bitno >= highest) break;
/* first set bit corresponds to the length word */
PolyWord *old = src->wordAddr(bitno); /* Old object address */
PolyObject *obj = (PolyObject*)(old+1);
POLYUNSIGNED L = obj->LengthWord();
ASSERT (OBJ_IS_LENGTH(L));
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L) + 1 ;/* Length of allocation (including length word) */
bitno += n;
// Find a mutable space for the mutable objects and an immutable space for
// the immutables. We copy objects into earlier spaces or within its own
// space but we don't copy an object to a later space. This avoids the
// risk of copying an object multiple times. Previously this copied objects
// into later spaces but that doesn't work well if we have converted old
// saved state segments into local areas. It's much better to delete them
// if possible.
bool isMutable = OBJ_IS_MUTABLE_OBJECT(L);
LocalMemSpace *destSpace = isMutable || immutableDest == 0 ? mutableDest : immutableDest;
PolyWord *newp = FindFreeAndAllocate(destSpace, (src == destSpace) ? bitno : 0, n);
if (newp == 0 && src != destSpace)
{
// See if we can find a different space.
// N.B. FindNextSpace side-effects mutableDest/immutableDest to give the next space.
if (FindNextSpace(src, isMutable ? &mutableDest : &immutableDest, isMutable, id))
{
bitno -= n; // Redo this object
continue;
}
// else just leave it
}
if (newp == 0) /* no room */
{
// We're not going to move this object
// Update src->upperAllocPtr, so the old object doesn't get trampled.
if (old < src->upperAllocPtr)
src->upperAllocPtr = old;
// Previously this continued compressing to try to make space available
// on the next GC. Normally full GCs are infrequent so the chances are
// that at the next GC other data will have been freed. Just stop at
// this point.
// However if we're compressing a mutable area and there is immutable
// data in it we should move those out because the mutable area is scanned
// on every partial GC.
if (! src->isMutable || src->i_marked == 0)
break;
}
else
{
PolyObject *destAddress = (PolyObject*)(newp+1);
obj->SetForwardingPtr(destAddress);
CopyObjectToNewAddress(obj, destAddress, L);
if (debugOptions & DEBUG_GC_DETAIL)
Log("GC: Copy: %p %lu %u -> %p\n", obj, OBJ_OBJECT_LENGTH(L),
GetTypeBits(L), destAddress);
}
}
if (mutableDest == src)
mutableDest = 0;
if (immutableDest == src)
immutableDest = 0;
}
}
