static void RestoreLengthWords(DepthVector *vec)
{
// Restore the length words.
Item *itemVec = vec->vector;
for (POLYUNSIGNED i = 0; i < vec->nitems; i++)
{
itemVec[i].pt->SetLengthWord(itemVec[i].L); // restore genuine length word
ASSERT (OBJ_IS_LENGTH(itemVec[i].pt->LengthWord()));
}
}
void DoCheckObject (const PolyObject *base, POLYUNSIGNED L)
{
PolyWord *pt = (PolyWord*)base;
CheckAddress(pt);
MemSpace *space = gMem.SpaceForAddress(pt-1);
if (space == 0)
Crash ("Bad pointer 0x%08" PRIxPTR " found", (uintptr_t)pt);
ASSERT (OBJ_IS_LENGTH(L));
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L);
if (n == 0) return;
ASSERT (n > 0);
ASSERT(pt-1 >= space->bottom && pt+n <= space->top);
byte flags = GetTypeBits(L); /* discards GC flag and mutable bit */
if (flags == F_BYTE_OBJ) /* possibly signed byte object */
return; /* Nothing more to do */
if (flags == F_CODE_OBJ) /* code object */
{
ScanCheckAddress checkAddr;
/* We flush the instruction cache here in case we change any of the
instructions when we update addresses. */
machineDependent->FlushInstructionCache(pt, (n + 1) * sizeof(PolyWord));
machineDependent->ScanConstantsWithinCode((PolyObject *)base, (PolyObject *)base, n, &checkAddr);
/* Skip to the constants. */
base->GetConstSegmentForCode(n, pt, n);
}
else if (flags == F_CLOSURE_OBJ)
{
n -= sizeof(PolyObject*) / sizeof(PolyWord);
pt += sizeof(PolyObject*) / sizeof(PolyWord);
}
else ASSERT (flags == 0); /* ordinary word object */
while (n--) DoCheck (*pt++);
}
// This is called via ScanAddressesInRegion to process the permanent mutables. It is
// also called from ScanObjectAddress to process root addresses.
// It processes all the addresses reachable from the object.
void RecursiveScan::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
if (OBJ_IS_BYTE_OBJECT(lengthWord))
{
Completed(obj);
return;
}
while (true)
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
// Get the length and base address. N.B. If this is a code segment
// these will be side-effected by GetConstSegmentForCode.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
PolyWord *baseAddr = (PolyWord*)obj;
if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// It's better to process the whole code object in one go.
ScanAddress::ScanAddressesInObject(obj, lengthWord);
length = 0; // Finished
}
ASSERT(! OBJ_IS_BYTE_OBJECT(lengthWord)); // Check - remove this later
// else it's a normal object,
// If there are only two addresses in this cell that need to be
// followed we follow them immediately and treat this cell as done.
// If there are more than two we push the address of this cell on
// the stack, follow the first address and then rescan it. That way
// list cells are processed once only but we don't overflow the
// stack by pushing all the addresses in a very large vector.
PolyWord *endWord = baseAddr + length;
PolyObject *firstWord = 0;
PolyObject *secondWord = 0;
while (baseAddr != endWord)
{
PolyWord wordAt = *baseAddr;
if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0))
{
// Normal address. We can have words of all zeros at least in the
// situation where we have a partially constructed code segment where
// the constants at the end of the code have not yet been filled in.
if (TestForScan(baseAddr)) // Test value at baseAddr (may side-effect it)
{
PolyObject *wObj = (*baseAddr).AsObjPtr();
if (wObj->IsByteObject())
{
// Can do this now - don't need to push it
MarkAsScanning(wObj);
Completed(wObj);
}
else if (firstWord == 0)
{
firstWord = wObj;
// We mark the word immediately. We can have
// two words in an object that are the same
// and we don't want to process it again.
MarkAsScanning(firstWord);
}
else if (secondWord == 0)
secondWord = wObj;
else break; // More than two words.
}
}
else if (wordAt.IsCodePtr())
{
// If we're processing the constant area of a code segment this could
// be a code address.
PolyObject *oldObject = ObjCodePtrToPtr(wordAt.AsCodePtr());
// Calculate the byte offset of this value within the code object.
POLYUNSIGNED offset = wordAt.AsCodePtr() - (byte*)oldObject;
wordAt = oldObject;
bool test = TestForScan(&wordAt);
// TestForScan may side-effect the word.
PolyObject *newObject = wordAt.AsObjPtr();
wordAt = PolyWord::FromCodePtr((byte*)newObject + offset);
if (wordAt != *baseAddr)
*baseAddr = wordAt;
if (test)
{
if (firstWord == 0)
{
firstWord = newObject;
MarkAsScanning(firstWord);
}
else if (secondWord == 0)
secondWord = newObject;
else break;
}
}
baseAddr++;
}
if (baseAddr == endWord)
{
// We have done everything except possibly firstWord and secondWord.
Completed(obj);
if (secondWord != 0)
{
MarkAsScanning(secondWord);
// Put this on the stack. If this is a list node we will be
// pushing the tail.
PushToStack(secondWord);
}
}
else // Put this back on the stack while we process the first word
PushToStack(obj);
if (firstWord != 0)
// Process it immediately.
obj = firstWord;
else if (StackIsEmpty())
return;
else
obj = PopFromStack();
lengthWord = obj->LengthWord();
}
}
// General purpose object processor, Processes all the addresses in an object.
// Handles the various kinds of object that may contain addresses.
void ScanAddress::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
do
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
if (OBJ_IS_BYTE_OBJECT(lengthWord))
return; /* Nothing more to do */
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
PolyWord *baseAddr = (PolyWord*)obj;
if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// Scan constants within the code.
machineDependent->ScanConstantsWithinCode(obj, obj, length, this);
// Skip to the constants and get ready to scan them.
obj->GetConstSegmentForCode(length, baseAddr, length);
} // else it's a normal object,
PolyWord *endWord = baseAddr + length;
// We want to minimise the actual recursion we perform so we try to
// use tail recursion if we can. We first scan from the end and
// remove any words that don't need recursion.
POLYUNSIGNED lastLengthWord = 0;
while (endWord != baseAddr)
{
PolyWord wordAt = endWord[-1];
if (IS_INT(wordAt) || wordAt == PolyWord::FromUnsigned(0))
endWord--; // Don't need to look at this.
else if ((lastLengthWord = ScanAddressAt(endWord-1)) != 0)
// We need to process this one
break;
else endWord--; // We're not interested in this.
}
if (endWord == baseAddr)
return; // We've done everything.
// There is at least one word that needs to be processed, the
// one at endWord-1.
// Now process from the beginning forward to see if there are
// any words before this that need to be handled. This way we are more
// likely to handle the head of a list by recursion and the
// tail by looping (tail recursion).
while (baseAddr < endWord-1)
{
PolyWord wordAt = *baseAddr;
if (IS_INT(wordAt) || wordAt == PolyWord::FromUnsigned(0))
baseAddr++; // Don't need to look at this.
else
{
POLYUNSIGNED lengthWord = ScanAddressAt(baseAddr);
if (lengthWord != 0)
{
wordAt = *baseAddr; // Reload because it may have been side-effected
// We really have to process this recursively.
if (wordAt.IsCodePtr())
ScanAddressesInObject(ObjCodePtrToPtr(wordAt.AsCodePtr()), lengthWord);
else
ScanAddressesInObject(wordAt.AsObjPtr(), lengthWord);
baseAddr++;
}
else baseAddr++;
}
}
// Finally process the last word we found that has to be processed.
// Do this by looping rather than recursion.
PolyWord wordAt = *baseAddr; // Last word to do.
// This must be an address
if (wordAt.IsCodePtr())
obj = ObjCodePtrToPtr(wordAt.AsCodePtr());
else
obj = wordAt.AsObjPtr();
lengthWord = lastLengthWord;
} while(1);
}
// We use _OBJ_GC_MARK to detect when we have visited a cell but not yet
// computed the depth. We have to be careful that this bit is removed
// before we finish in the case that we run out of memory and throw an
// exception. PushToStack may throw the exception if the stack needs to
// grow.
POLYUNSIGNED ProcessAddToVector::AddObjectsToDepthVectors(PolyWord old)
{
// If this is a tagged integer or an IO pointer that's simply a constant.
if (old.IsTagged() || old == PolyWord::FromUnsigned(0))
return 0;
MemSpace *space = gMem.SpaceForAddress(old.AsAddress());
if (space == 0 || space->spaceType == ST_IO)
return 0;
PolyObject *obj = old.AsObjPtr();
POLYUNSIGNED L = obj->LengthWord();
if (OBJ_IS_DEPTH(L)) // tombstone contains genuine depth or 0.
return OBJ_GET_DEPTH(L);
if (obj->LengthWord() & _OBJ_GC_MARK)
return 0; // Marked but not yet scanned. Circular structure.
ASSERT (OBJ_IS_LENGTH(L));
if (obj->IsMutable())
{
// Mutable data in the local or permanent areas
if (! obj->IsByteObject())
{
// Add it to the vector so we will update any addresses it contains.
m_parent->AddToVector(0, L, old.AsObjPtr());
// and follow any addresses to try to merge those.
PushToStack(obj);
obj->SetLengthWord(L | _OBJ_GC_MARK); // To prevent rescan
}
return 0; // Level is zero
}
if (space->spaceType == ST_PERMANENT &&
((PermanentMemSpace*)space)->hierarchy == 0)
{
// Immutable data in the permanent area can't be merged
// because it's read only. We need to follow the addresses
// because they may point to mutable areas containing data
// that can be. A typical case is the root function pointing
// at the global name table containing new declarations.
Bitmap *bm = &((PermanentMemSpace*)space)->shareBitmap;
if (! bm->TestBit((PolyWord*)obj - space->bottom))
{
bm->SetBit((PolyWord*)obj - space->bottom);
if (! obj->IsByteObject())
PushToStack(obj);
}
return 0;
}
/* There's a problem sharing code objects if they have relative calls/jumps
in them to other code. The code of two functions may be identical (e.g.
they both call functions 100 bytes ahead) and so they will appear the
same but if the functions they jump to are different they are actually
different. For that reason we don't share code segments. DCJM 4/1/01 */
if (obj->IsCodeObject())
{
// We want to update addresses in the code segment.
m_parent->AddToVector(0, L, old.AsObjPtr());
PushToStack(obj);
obj->SetLengthWord(L | _OBJ_GC_MARK); // To prevent rescan
return 0;
}
// Byte objects always have depth 1 and can't contain addresses.
if (obj->IsByteObject())
{
m_parent->AddToVector (1, L, old.AsObjPtr());// add to vector at correct depth
obj->SetLengthWord(OBJ_SET_DEPTH(1));
return 1;
}
ASSERT(OBJ_IS_WORD_OBJECT(L)); // That leaves immutable data objects.
PushToStack(obj);
obj->SetLengthWord(L | _OBJ_GC_MARK); // To prevent rescan
return 0;
}
// Merge cells with the same contents.
POLYUNSIGNED DepthVector::MergeSameItems()
{
DepthVector *v = this;
POLYUNSIGNED N = v->nitems;
Item *itemVec = v->vector;
POLYUNSIGNED n = 0;
POLYUNSIGNED i = 0;
while (i < N)
{
PolyObject *bestShare = 0; // Candidate to share.
MemSpace *bestSpace = 0;
POLYUNSIGNED j;
for (j = i; j < N; j++)
{
ASSERT (OBJ_IS_DEPTH(itemVec[i].pt->LengthWord()));
// Search for identical objects. Don't bother to compare it with itself.
if (i != j && CompareItems (& itemVec[i], & itemVec[j]) != 0) break;
// The order of sharing is significant.
// Choose an object in the permanent memory if that is available.
// This is necessary to retain the invariant that no object in
// the permanent memory points to an object in the temporary heap.
// (There may well be pointers to this object elsewhere in the permanent
// heap).
// Choose the lowest hierarchy value for preference since that
// may reduce the size of saved state when resaving already saved
// data.
// If we can't find a permanent space choose a space that isn't
// an allocation space. Otherwise we could break the invariant
// that immutable areas never point into the allocation area.
MemSpace *space = gMem.SpaceForAddress(itemVec[j].pt);
if (bestSpace == 0)
{
bestShare = itemVec[j].pt;
bestSpace = space;
}
else if (bestSpace->spaceType == ST_PERMANENT)
{
// Only update if the current space is also permanent and a lower hierarchy
if (space->spaceType == ST_PERMANENT &&
((PermanentMemSpace *)space)->hierarchy < ((PermanentMemSpace *)bestSpace)->hierarchy)
{
bestShare = itemVec[j].pt;
bestSpace = space;
}
}
else if (bestSpace->spaceType == ST_LOCAL)
{
// Update if the current space is not an allocation space
if (space->spaceType != ST_LOCAL || ! ((LocalMemSpace*)space)->allocationSpace)
{
bestShare = itemVec[j].pt;
bestSpace = space;
}
}
}
POLYUNSIGNED k = j; // Remember the first object that didn't match.
//.For each identical object set all but the one we want to point to
// the shared object.
for (j = i; j < k; j++)
{
ASSERT (OBJ_IS_DEPTH(itemVec[j].pt->LengthWord()));
if (itemVec[j].pt == bestShare)
{
// This is the common object.
bestShare->SetLengthWord(itemVec[j].L); // restore genuine length word
ASSERT (OBJ_IS_LENGTH(bestShare->LengthWord()));
}
else
{
itemVec[j].pt->SetForwardingPtr(bestShare); /* an indirection */
ASSERT (itemVec[j].pt->ContainsForwardingPtr());
n++;
}
}
ASSERT(! OBJ_IS_DEPTH(itemVec[i].pt->LengthWord()));
i = k;
}
return n;
}
// This is called via ScanAddressesInRegion to process the permanent mutables. It is
// also called from ScanObjectAddress to process root addresses.
// It processes all the addresses reachable from the object.
void MTGCProcessMarkPointers::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
if (OBJ_IS_BYTE_OBJECT(lengthWord))
return;
while (true)
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
// Get the length and base address. N.B. If this is a code segment
// these will be side-effected by GetConstSegmentForCode.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
if (OBJ_IS_WEAKREF_OBJECT(lengthWord))
{
// Special case.
ASSERT(OBJ_IS_MUTABLE_OBJECT(lengthWord)); // Should be a mutable.
ASSERT(OBJ_IS_WORD_OBJECT(lengthWord)); // Should be a plain object.
// We need to mark the "SOME" values in this object but we don't mark
// the references contained within the "SOME".
PolyWord *baseAddr = (PolyWord*)obj;
// Mark every word but ignore the result.
for (POLYUNSIGNED i = 0; i < length; i++)
(void)MarkAndTestForScan(baseAddr+i);
// We've finished with this.
length = 0;
}
else if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// It's better to process the whole code object in one go.
ScanAddress::ScanAddressesInObject(obj, lengthWord);
length = 0; // Finished
}
// else it's a normal object,
// If there are only two addresses in this cell that need to be
// followed we follow them immediately and treat this cell as done.
// If there are more than two we push the address of this cell on
// the stack, follow the first address and then rescan it. That way
// list cells are processed once only but we don't overflow the
// stack by pushing all the addresses in a very large vector.
PolyWord *baseAddr = (PolyWord*)obj;
PolyWord *endWord = baseAddr + length;
PolyObject *firstWord = 0;
PolyObject *secondWord = 0;
PolyWord *restartAddr = 0;
if (obj == largeObjectCache[locPtr].base)
{
baseAddr = largeObjectCache[locPtr].current;
ASSERT(baseAddr > (PolyWord*)obj && baseAddr < ((PolyWord*)obj)+length);
if (locPtr == 0) locPtr = LARGECACHE_SIZE-1; else locPtr--;
}
while (baseAddr != endWord)
{
PolyWord wordAt = *baseAddr;
if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0))
{
// Normal address. We can have words of all zeros at least in the
// situation where we have a partially constructed code segment where
// the constants at the end of the code have not yet been filled in.
if (TestForScan(baseAddr))
{
if (firstWord == 0)
firstWord = baseAddr->AsObjPtr();
else if (secondWord == 0)
{
// If we need to rescan because there are three or more words to do
// this is the place we need to restart (or the start of the cell if it's
// small).
restartAddr = baseAddr;
secondWord = baseAddr->AsObjPtr();
}
else break; // More than two words.
}
}
else if (wordAt.IsCodePtr())
{
// If we're processing the constant area of a code segment this could
// be a code address.
// Check that this is actually an address. If we have had a bad pointer
// earlier we may treat some length fields as values.
ASSERT(gMem.SpaceForAddress(wordAt.AsCodePtr()) != 0);
PolyObject *oldObject = ObjCodePtrToPtr(wordAt.AsCodePtr());
// Calculate the byte offset of this value within the code object.
POLYUNSIGNED offset = wordAt.AsCodePtr() - (byte*)oldObject;
wordAt = oldObject;
bool test = TestForScan(&wordAt);
// If we've changed it because we had a left-over forwarding pointer
// we need to update the original.
PolyObject *newObject = wordAt.AsObjPtr();
wordAt = PolyWord::FromCodePtr((byte*)newObject + offset);
if (wordAt != *baseAddr)
*baseAddr = wordAt;
if (test)
{
if (firstWord == 0)
firstWord = newObject;
else if (secondWord == 0)
{
restartAddr = baseAddr;
secondWord = newObject;
}
else break;
}
}
baseAddr++;
}
if (baseAddr != endWord)
// Put this back on the stack while we process the first word
PushToStack(obj, length < largeObjectSize ? 0 : restartAddr, length);
else if (secondWord != 0)
{
// Mark it now because we will process it.
secondWord->SetLengthWord(secondWord->LengthWord() | _OBJ_GC_MARK);
// Put this on the stack. If this is a list node we will be
// pushing the tail.
PushToStack(secondWord);
}
if (firstWord != 0)
{
// Mark it and process it immediately.
firstWord->SetLengthWord(firstWord->LengthWord() | _OBJ_GC_MARK);
obj = firstWord;
}
else if (msp == 0)
{
markStack[msp] = 0; // Really finished
return;
}
else
{
// Clear the item above the top. This really is finished.
if (msp < MARK_STACK_SIZE) markStack[msp] = 0;
// Pop the item from the stack but don't overwrite it yet.
// This allows another thread to steal it if there really
// is nothing else to do. This is only really important
// for large objects.
obj = markStack[--msp]; // Pop something.
}
lengthWord = obj->LengthWord();
}
}
// Copy objects from the source space into an earlier space or up within the
// current space.
static void copyAllData(GCTaskId *id, void * /*arg1*/, void * /*arg2*/)
{
LocalMemSpace *mutableDest = 0, *immutableDest = 0;
for (std::vector<LocalMemSpace*>::reverse_iterator i = gMem.lSpaces.rbegin(); i != gMem.lSpaces.rend(); i++)
{
LocalMemSpace *src = *i;
if (src->spaceOwner == 0)
{
PLocker lock(©Lock);
if (src->spaceOwner == 0)
src->spaceOwner = id;
else continue;
}
else if (src->spaceOwner != id)
continue;
if (debugOptions & DEBUG_GC_ENHANCED)
Log("GC: Copy: copying area %p (thread %p) %s \n", src, id, src->spaceTypeString());
// We start at fullGCLowerLimit which is the lowest marked object in the heap
// N.B. It's essential that the first set bit at or above this corresponds
// to the length word of a real object.
uintptr_t bitno = src->wordNo(src->fullGCLowerLimit);
// Set the limit to the top so we won't rescan this. That can
// only happen if copying takes a very short time and the same
// thread runs multiple tasks.
src->fullGCLowerLimit = src->top;
// src->highest is the bit position that corresponds to the top of
// generation we're copying.
uintptr_t highest = src->wordNo(src->top);
for (;;)
{
if (bitno >= highest) break;
/* SPF version; Invariant: 0 < highest - bitno */
bitno += src->bitmap.CountZeroBits(bitno, highest - bitno);
if (bitno >= highest) break;
/* first set bit corresponds to the length word */
PolyWord *old = src->wordAddr(bitno); /* Old object address */
PolyObject *obj = (PolyObject*)(old+1);
POLYUNSIGNED L = obj->LengthWord();
ASSERT (OBJ_IS_LENGTH(L));
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L) + 1 ;/* Length of allocation (including length word) */
bitno += n;
// Find a mutable space for the mutable objects and an immutable space for
// the immutables. We copy objects into earlier spaces or within its own
// space but we don't copy an object to a later space. This avoids the
// risk of copying an object multiple times. Previously this copied objects
// into later spaces but that doesn't work well if we have converted old
// saved state segments into local areas. It's much better to delete them
// if possible.
bool isMutable = OBJ_IS_MUTABLE_OBJECT(L);
LocalMemSpace *destSpace = isMutable || immutableDest == 0 ? mutableDest : immutableDest;
PolyWord *newp = FindFreeAndAllocate(destSpace, (src == destSpace) ? bitno : 0, n);
if (newp == 0 && src != destSpace)
{
// See if we can find a different space.
// N.B. FindNextSpace side-effects mutableDest/immutableDest to give the next space.
if (FindNextSpace(src, isMutable ? &mutableDest : &immutableDest, isMutable, id))
{
bitno -= n; // Redo this object
continue;
}
// else just leave it
}
if (newp == 0) /* no room */
{
// We're not going to move this object
// Update src->upperAllocPtr, so the old object doesn't get trampled.
if (old < src->upperAllocPtr)
src->upperAllocPtr = old;
// Previously this continued compressing to try to make space available
// on the next GC. Normally full GCs are infrequent so the chances are
// that at the next GC other data will have been freed. Just stop at
// this point.
// However if we're compressing a mutable area and there is immutable
// data in it we should move those out because the mutable area is scanned
// on every partial GC.
if (! src->isMutable || src->i_marked == 0)
break;
}
else
{
PolyObject *destAddress = (PolyObject*)(newp+1);
obj->SetForwardingPtr(destAddress);
CopyObjectToNewAddress(obj, destAddress, L);
if (debugOptions & DEBUG_GC_DETAIL)
Log("GC: Copy: %p %lu %u -> %p\n", obj, OBJ_OBJECT_LENGTH(L),
GetTypeBits(L), destAddress);
}
}
if (mutableDest == src)
mutableDest = 0;
if (immutableDest == src)
immutableDest = 0;
}
}
