static void SetBitmaps(LocalMemSpace *space, PolyWord *pt, PolyWord *top)
{
while (pt < top)
{
PolyObject *obj = (PolyObject*)++pt;
// If it has been copied by a minor collection skip it
if (obj->ContainsForwardingPtr())
{
obj = FollowForwarding(obj);
ASSERT(obj->ContainsNormalLengthWord());
pt += obj->Length();
}
else
{
POLYUNSIGNED L = obj->LengthWord();
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L);
if (L & _OBJ_GC_MARK)
{
obj->SetLengthWord(L & ~(_OBJ_GC_MARK));
POLYUNSIGNED bitno = space->wordNo(pt);
space->bitmap.SetBits(bitno - 1, n + 1);
if (OBJ_IS_MUTABLE_OBJECT(L))
space->m_marked += n + 1;
else
space->i_marked += n + 1;
if ((PolyWord*)obj <= space->fullGCLowerLimit)
space->fullGCLowerLimit = (PolyWord*)obj-1;
if (OBJ_IS_WEAKREF_OBJECT(L))
{
// Add this to the limits for the containing area.
PolyWord *baseAddr = (PolyWord*)obj;
PolyWord *startAddr = baseAddr-1; // Must point AT length word.
PolyWord *endObject = baseAddr + n;
if (startAddr < space->lowestWeak) space->lowestWeak = startAddr;
if (endObject > space->highestWeak) space->highestWeak = endObject;
}
}
pt += n;
}
}
}
// Deal with weak objects
void MTGCCheckWeakRef::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED L)
{
if (! OBJ_IS_WEAKREF_OBJECT(L)) return;
ASSERT(OBJ_IS_MUTABLE_OBJECT(L)); // Should be a mutable.
ASSERT(OBJ_IS_WORD_OBJECT(L)); // Should be a plain object.
// See if any of the SOME objects contain unreferenced refs.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(L);
PolyWord *baseAddr = (PolyWord*)obj;
for (POLYUNSIGNED i = 0; i < length; i++)
{
PolyWord someAddr = baseAddr[i];
if (someAddr.IsDataPtr())
{
LocalMemSpace *someSpace = gMem.LocalSpaceForAddress(someAddr.AsAddress());
if (someSpace != 0)
{
PolyObject *someObj = someAddr.AsObjPtr();
// If this is a weak object the SOME value may refer to an unreferenced
// ref. If so we have to set this entry to NONE. For safety we also
// set the contents of the SOME to TAGGED(0).
ASSERT(someObj->Length() == 1 && someObj->IsWordObject()); // Should be a SOME node.
PolyWord refAddress = someObj->Get(0);
LocalMemSpace *space = gMem.LocalSpaceForAddress(refAddress.AsAddress());
if (space != 0)
// If the ref is permanent it's always there.
{
POLYUNSIGNED new_bitno = space->wordNo(refAddress.AsStackAddr());
if (! space->bitmap.TestBit(new_bitno))
{
// It wasn't marked so it's otherwise unreferenced.
baseAddr[i] = TAGGED(0); // Set it to NONE.
someObj->Set(0, TAGGED(0)); // For safety.
convertedWeak = true;
}
}
}
}
}
}
void GetSharing::Completed(PolyObject *obj)
{
// We mustn't include cells in the permanent area.
// We scan the permanent mutable areas for local addresses
// but we mustn't add the cells themselves. Normally they
// will be mutable so would be ignored but cells that have been
// locked will now be immutable. The test in TestForScan is bypassed
// by ScanAddressesInRegion.
PolyWord *lengthWord = ((PolyWord*)obj) - 1;
if (gMem.LocalSpaceForAddress(lengthWord) == 0)
return;
POLYUNSIGNED L = obj->LengthWord();
// We have tables for word objects and byte objects
// We chain entries together using the length word so it
// is important that we only do this for objects that
// have no other bits in the header, such as the sign bit.
if ((L & _OBJ_PRIVATE_FLAGS_MASK) == 0)
{
POLYUNSIGNED length = obj->Length();
if (length < NUM_WORD_VECTORS)
wordVectors[length].AddToVector(obj, length);
else largeWordCount++;
wordAdded++;
}
else if ((L & _OBJ_PRIVATE_FLAGS_MASK) == _OBJ_BYTE_OBJ)
{
POLYUNSIGNED length = obj->Length();
if (length < NUM_BYTE_VECTORS)
byteVectors[length].AddToVector(obj, length);
else largeByteCount++;
byteAdded++;
}
else if (! OBJ_IS_CODE_OBJECT(L) && ! OBJ_IS_MUTABLE_OBJECT(L))
excludedCount++; // Code and mutables can't be shared - see what could be
// TODO: We don't attempt to share closure cells in 32-in-64.
}
// This is called via ScanAddressesInRegion to process the permanent mutables. It is
// also called from ScanObjectAddress to process root addresses.
// It processes all the addresses reachable from the object.
void MTGCProcessMarkPointers::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
if (OBJ_IS_BYTE_OBJECT(lengthWord))
return;
while (true)
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
// Get the length and base address. N.B. If this is a code segment
// these will be side-effected by GetConstSegmentForCode.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
if (OBJ_IS_WEAKREF_OBJECT(lengthWord))
{
// Special case.
ASSERT(OBJ_IS_MUTABLE_OBJECT(lengthWord)); // Should be a mutable.
ASSERT(OBJ_IS_WORD_OBJECT(lengthWord)); // Should be a plain object.
// We need to mark the "SOME" values in this object but we don't mark
// the references contained within the "SOME".
PolyWord *baseAddr = (PolyWord*)obj;
// Mark every word but ignore the result.
for (POLYUNSIGNED i = 0; i < length; i++)
(void)MarkAndTestForScan(baseAddr+i);
// We've finished with this.
length = 0;
}
else if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// It's better to process the whole code object in one go.
ScanAddress::ScanAddressesInObject(obj, lengthWord);
length = 0; // Finished
}
// else it's a normal object,
// If there are only two addresses in this cell that need to be
// followed we follow them immediately and treat this cell as done.
// If there are more than two we push the address of this cell on
// the stack, follow the first address and then rescan it. That way
// list cells are processed once only but we don't overflow the
// stack by pushing all the addresses in a very large vector.
PolyWord *baseAddr = (PolyWord*)obj;
PolyWord *endWord = baseAddr + length;
PolyObject *firstWord = 0;
PolyObject *secondWord = 0;
PolyWord *restartAddr = 0;
if (obj == largeObjectCache[locPtr].base)
{
baseAddr = largeObjectCache[locPtr].current;
ASSERT(baseAddr > (PolyWord*)obj && baseAddr < ((PolyWord*)obj)+length);
if (locPtr == 0) locPtr = LARGECACHE_SIZE-1; else locPtr--;
}
while (baseAddr != endWord)
{
PolyWord wordAt = *baseAddr;
if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0))
{
// Normal address. We can have words of all zeros at least in the
// situation where we have a partially constructed code segment where
// the constants at the end of the code have not yet been filled in.
if (TestForScan(baseAddr))
{
if (firstWord == 0)
firstWord = baseAddr->AsObjPtr();
else if (secondWord == 0)
{
// If we need to rescan because there are three or more words to do
// this is the place we need to restart (or the start of the cell if it's
// small).
restartAddr = baseAddr;
secondWord = baseAddr->AsObjPtr();
}
else break; // More than two words.
}
}
else if (wordAt.IsCodePtr())
{
// If we're processing the constant area of a code segment this could
// be a code address.
// Check that this is actually an address. If we have had a bad pointer
// earlier we may treat some length fields as values.
ASSERT(gMem.SpaceForAddress(wordAt.AsCodePtr()) != 0);
PolyObject *oldObject = ObjCodePtrToPtr(wordAt.AsCodePtr());
// Calculate the byte offset of this value within the code object.
POLYUNSIGNED offset = wordAt.AsCodePtr() - (byte*)oldObject;
wordAt = oldObject;
bool test = TestForScan(&wordAt);
// If we've changed it because we had a left-over forwarding pointer
// we need to update the original.
PolyObject *newObject = wordAt.AsObjPtr();
wordAt = PolyWord::FromCodePtr((byte*)newObject + offset);
if (wordAt != *baseAddr)
*baseAddr = wordAt;
if (test)
{
if (firstWord == 0)
firstWord = newObject;
else if (secondWord == 0)
{
restartAddr = baseAddr;
secondWord = newObject;
}
else break;
}
}
baseAddr++;
}
if (baseAddr != endWord)
// Put this back on the stack while we process the first word
PushToStack(obj, length < largeObjectSize ? 0 : restartAddr, length);
else if (secondWord != 0)
{
// Mark it now because we will process it.
secondWord->SetLengthWord(secondWord->LengthWord() | _OBJ_GC_MARK);
// Put this on the stack. If this is a list node we will be
// pushing the tail.
PushToStack(secondWord);
}
if (firstWord != 0)
{
// Mark it and process it immediately.
firstWord->SetLengthWord(firstWord->LengthWord() | _OBJ_GC_MARK);
obj = firstWord;
}
else if (msp == 0)
{
markStack[msp] = 0; // Really finished
return;
}
else
{
// Clear the item above the top. This really is finished.
if (msp < MARK_STACK_SIZE) markStack[msp] = 0;
// Pop the item from the stack but don't overwrite it yet.
// This allows another thread to steal it if there really
// is nothing else to do. This is only really important
// for large objects.
obj = markStack[--msp]; // Pop something.
}
lengthWord = obj->LengthWord();
}
}
// Copy objects from the source space into an earlier space or up within the
// current space.
static void copyAllData(GCTaskId *id, void * /*arg1*/, void * /*arg2*/)
{
LocalMemSpace *mutableDest = 0, *immutableDest = 0;
for (std::vector<LocalMemSpace*>::reverse_iterator i = gMem.lSpaces.rbegin(); i != gMem.lSpaces.rend(); i++)
{
LocalMemSpace *src = *i;
if (src->spaceOwner == 0)
{
PLocker lock(©Lock);
if (src->spaceOwner == 0)
src->spaceOwner = id;
else continue;
}
else if (src->spaceOwner != id)
continue;
if (debugOptions & DEBUG_GC_ENHANCED)
Log("GC: Copy: copying area %p (thread %p) %s \n", src, id, src->spaceTypeString());
// We start at fullGCLowerLimit which is the lowest marked object in the heap
// N.B. It's essential that the first set bit at or above this corresponds
// to the length word of a real object.
uintptr_t bitno = src->wordNo(src->fullGCLowerLimit);
// Set the limit to the top so we won't rescan this. That can
// only happen if copying takes a very short time and the same
// thread runs multiple tasks.
src->fullGCLowerLimit = src->top;
// src->highest is the bit position that corresponds to the top of
// generation we're copying.
uintptr_t highest = src->wordNo(src->top);
for (;;)
{
if (bitno >= highest) break;
/* SPF version; Invariant: 0 < highest - bitno */
bitno += src->bitmap.CountZeroBits(bitno, highest - bitno);
if (bitno >= highest) break;
/* first set bit corresponds to the length word */
PolyWord *old = src->wordAddr(bitno); /* Old object address */
PolyObject *obj = (PolyObject*)(old+1);
POLYUNSIGNED L = obj->LengthWord();
ASSERT (OBJ_IS_LENGTH(L));
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L) + 1 ;/* Length of allocation (including length word) */
bitno += n;
// Find a mutable space for the mutable objects and an immutable space for
// the immutables. We copy objects into earlier spaces or within its own
// space but we don't copy an object to a later space. This avoids the
// risk of copying an object multiple times. Previously this copied objects
// into later spaces but that doesn't work well if we have converted old
// saved state segments into local areas. It's much better to delete them
// if possible.
bool isMutable = OBJ_IS_MUTABLE_OBJECT(L);
LocalMemSpace *destSpace = isMutable || immutableDest == 0 ? mutableDest : immutableDest;
PolyWord *newp = FindFreeAndAllocate(destSpace, (src == destSpace) ? bitno : 0, n);
if (newp == 0 && src != destSpace)
{
// See if we can find a different space.
// N.B. FindNextSpace side-effects mutableDest/immutableDest to give the next space.
if (FindNextSpace(src, isMutable ? &mutableDest : &immutableDest, isMutable, id))
{
bitno -= n; // Redo this object
continue;
}
// else just leave it
}
if (newp == 0) /* no room */
{
// We're not going to move this object
// Update src->upperAllocPtr, so the old object doesn't get trampled.
if (old < src->upperAllocPtr)
src->upperAllocPtr = old;
// Previously this continued compressing to try to make space available
// on the next GC. Normally full GCs are infrequent so the chances are
// that at the next GC other data will have been freed. Just stop at
// this point.
// However if we're compressing a mutable area and there is immutable
// data in it we should move those out because the mutable area is scanned
// on every partial GC.
if (! src->isMutable || src->i_marked == 0)
break;
}
else
{
PolyObject *destAddress = (PolyObject*)(newp+1);
obj->SetForwardingPtr(destAddress);
CopyObjectToNewAddress(obj, destAddress, L);
if (debugOptions & DEBUG_GC_DETAIL)
Log("GC: Copy: %p %lu %u -> %p\n", obj, OBJ_OBJECT_LENGTH(L),
GetTypeBits(L), destAddress);
}
}
if (mutableDest == src)
mutableDest = 0;
if (immutableDest == src)
immutableDest = 0;
}
}
