void SortVector::hashAndSortAllTask(GCTaskId*, void *a, void *b)
{
SortVector *s = (SortVector *)a;
// Hash the contents of the base object then sort them.
for (unsigned i = 0; i < 256; i++)
{
// Clear the entries in the hash table but not the sharing count.
s->processObjects[i].objList = ENDOFLIST;
s->processObjects[i].objCount = 0;
}
PolyObject *h = s->baseObject.objList;
POLYUNSIGNED bytes = OBJ_OBJECT_LENGTH(s->lengthWord)*sizeof(PolyWord);
while (h != ENDOFLIST)
{
PolyObject *next = h->GetForwardingPtr();
unsigned char hash = 0;
for (POLYUNSIGNED j = 0; j < bytes; j++)
hash += h->AsBytePtr()[j];
h->SetForwardingPtr(s->processObjects[hash].objList);
s->processObjects[hash].objList = h;
s->processObjects[hash].objCount++;
h = next;
}
s->SortData();
}
// Tests if this needs to be scanned. It marks it if it has not been marked
// unless it has to be scanned.
bool MTGCProcessMarkPointers::TestForScan(PolyWord *pt)
{
if ((*pt).IsTagged())
return false;
// This could contain a forwarding pointer if it points into an
// allocation area and has been moved by the minor GC.
// We have to be a little careful. Another thread could also
// be following any forwarding pointers here. However it's safe
// because they will update it with the same value.
PolyObject *obj = (*pt).AsObjPtr();
if (obj->ContainsForwardingPtr())
{
obj = FollowForwarding(obj);
*pt = obj;
}
if (gMem.LocalSpaceForAddress(obj) == 0)
return false; // Ignore it if it points to a permanent area
POLYUNSIGNED L = obj->LengthWord();
if (L & _OBJ_GC_MARK)
return false; // Already marked
if (debugOptions & DEBUG_GC_DETAIL)
Log("GC: Mark: %p %" POLYUFMT " %u\n", obj, OBJ_OBJECT_LENGTH(L), GetTypeBits(L));
if (OBJ_IS_BYTE_OBJECT(L))
{
obj->SetLengthWord(L | _OBJ_GC_MARK); // Mark it
return false; // We've done as much as we need
}
return true;
}
// Sort the entries in the hash table.
void SortVector::SortData()
{
for (unsigned j = 0; j < 256; j++)
{
ObjEntry *oentry = &processObjects[j];
// Sort this entry. If it's very small just process it now.
switch (oentry->objCount)
{
case 0: break; // Nothing there
case 1: // Singleton - just restore the length word
oentry->objList->SetLengthWord(lengthWord);
break;
case 2:
{
// Two items - process now
PolyObject *obj1 = oentry->objList;
PolyObject *obj2 = obj1->GetForwardingPtr();
obj1->SetLengthWord(lengthWord);
if (memcmp(obj1, obj2, OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord)) == 0)
{
shareWith(obj2, obj1);
oentry->shareCount++;
}
else obj2->SetLengthWord(lengthWord);
break;
}
default:
gpTaskFarm->AddWorkOrRunNow(sharingTask, this, oentry);
}
}
}
// The initial entry to process the roots. These may be RTS addresses or addresses in
// a thread stack. Also called recursively to process the addresses of constants in
// code segments. This is used in situations where a scanner may return the
// updated address of an object.
PolyObject *MTGCProcessMarkPointers::ScanObjectAddress(PolyObject *obj)
{
PolyWord val = obj;
LocalMemSpace *space = gMem.LocalSpaceForAddress(val.AsAddress());
if (space == 0)
return obj; // Ignore it if it points to a permanent area
// We may have a forwarding pointer if this has been moved by the
// minor GC.
if (obj->ContainsForwardingPtr())
{
obj = FollowForwarding(obj);
val = obj;
space = gMem.LocalSpaceForAddress(val.AsAddress());
}
ASSERT(obj->ContainsNormalLengthWord());
POLYUNSIGNED L = obj->LengthWord();
if (L & _OBJ_GC_MARK)
return obj; // Already marked
obj->SetLengthWord(L | _OBJ_GC_MARK); // Mark it
if (profileMode == kProfileLiveData || (profileMode == kProfileLiveMutables && obj->IsMutable()))
AddObjectProfile(obj);
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L);
if (debugOptions & DEBUG_GC_DETAIL)
Log("GC: Mark: %p %" POLYUFMT " %u\n", obj, n, GetTypeBits(L));
if (OBJ_IS_BYTE_OBJECT(L))
return obj;
// If we already have something on the stack we must being called
// recursively to process a constant in a code segment. Just push
// it on the stack and let the caller deal with it.
if (msp != 0)
PushToStack(obj); // Can't check this because it may have forwarding ptrs.
else
{
MTGCProcessMarkPointers::ScanAddressesInObject(obj, L);
// We can only check after we've processed it because if we
// have addresses left over from an incomplete partial GC they
// may need to forwarded.
CheckObject (obj);
}
return obj;
}
static void SetBitmaps(LocalMemSpace *space, PolyWord *pt, PolyWord *top)
{
while (pt < top)
{
PolyObject *obj = (PolyObject*)++pt;
// If it has been copied by a minor collection skip it
if (obj->ContainsForwardingPtr())
{
obj = FollowForwarding(obj);
ASSERT(obj->ContainsNormalLengthWord());
pt += obj->Length();
}
else
{
POLYUNSIGNED L = obj->LengthWord();
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L);
if (L & _OBJ_GC_MARK)
{
obj->SetLengthWord(L & ~(_OBJ_GC_MARK));
POLYUNSIGNED bitno = space->wordNo(pt);
space->bitmap.SetBits(bitno - 1, n + 1);
if (OBJ_IS_MUTABLE_OBJECT(L))
space->m_marked += n + 1;
else
space->i_marked += n + 1;
if ((PolyWord*)obj <= space->fullGCLowerLimit)
space->fullGCLowerLimit = (PolyWord*)obj-1;
if (OBJ_IS_WEAKREF_OBJECT(L))
{
// Add this to the limits for the containing area.
PolyWord *baseAddr = (PolyWord*)obj;
PolyWord *startAddr = baseAddr-1; // Must point AT length word.
PolyWord *endObject = baseAddr + n;
if (startAddr < space->lowestWeak) space->lowestWeak = startAddr;
if (endObject > space->highestWeak) space->highestWeak = endObject;
}
}
pt += n;
}
}
}
// Comparison function used for sorting and also to test whether
// two cells can be merged.
int DepthVector::CompareItems(const Item *a, const Item *b)
{
PolyObject *x = a->pt;
PolyObject *y = b->pt;
// POLYUNSIGNED A = x->LengthWord();
// POLYUNSIGNED B = y->LengthWord();
// ASSERT (OBJ_IS_DEPTH(A));
// ASSERT (OBJ_IS_DEPTH(B));
// ASSERT (A == B); // Should be the same depth.
// ASSERT (OBJ_IS_LENGTH(a->L));
// ASSERT (OBJ_IS_LENGTH(b->L));
if (a->L > b->L) return 1; // These tests include the flag bits
if (a->L < b->L) return -1;
// Return simple bitwise equality.
return memcmp(x, y, OBJ_OBJECT_LENGTH(a->L)*sizeof(PolyWord));
}
void DoCheckObject (const PolyObject *base, POLYUNSIGNED L)
{
PolyWord *pt = (PolyWord*)base;
CheckAddress(pt);
MemSpace *space = gMem.SpaceForAddress(pt-1);
if (space == 0)
Crash ("Bad pointer 0x%08" PRIxPTR " found", (uintptr_t)pt);
ASSERT (OBJ_IS_LENGTH(L));
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L);
if (n == 0) return;
ASSERT (n > 0);
ASSERT(pt-1 >= space->bottom && pt+n <= space->top);
byte flags = GetTypeBits(L); /* discards GC flag and mutable bit */
if (flags == F_BYTE_OBJ) /* possibly signed byte object */
return; /* Nothing more to do */
if (flags == F_CODE_OBJ) /* code object */
{
ScanCheckAddress checkAddr;
/* We flush the instruction cache here in case we change any of the
instructions when we update addresses. */
machineDependent->FlushInstructionCache(pt, (n + 1) * sizeof(PolyWord));
machineDependent->ScanConstantsWithinCode((PolyObject *)base, (PolyObject *)base, n, &checkAddr);
/* Skip to the constants. */
base->GetConstSegmentForCode(n, pt, n);
}
else if (flags == F_CLOSURE_OBJ)
{
n -= sizeof(PolyObject*) / sizeof(PolyWord);
pt += sizeof(PolyObject*) / sizeof(PolyWord);
}
else ASSERT (flags == 0); /* ordinary word object */
while (n--) DoCheck (*pt++);
}
// Copy a cell to its new address.
void CopyObjectToNewAddress(PolyObject *srcAddress, PolyObject *destAddress, POLYUNSIGNED L)
{
destAddress->SetLengthWord(L); /* copy length word */
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L);
// Unroll loop for most common cases.
switch (n)
{
default:
memcpy(destAddress, srcAddress, n * sizeof(PolyWord));
break;
case 4:
destAddress->Set(3, srcAddress->Get(3));
case 3:
destAddress->Set(2, srcAddress->Get(2));
case 2:
destAddress->Set(1, srcAddress->Get(1));
case 1:
destAddress->Set(0, srcAddress->Get(0));
}
}
// Deal with weak objects
void MTGCCheckWeakRef::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED L)
{
if (! OBJ_IS_WEAKREF_OBJECT(L)) return;
ASSERT(OBJ_IS_MUTABLE_OBJECT(L)); // Should be a mutable.
ASSERT(OBJ_IS_WORD_OBJECT(L)); // Should be a plain object.
// See if any of the SOME objects contain unreferenced refs.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(L);
PolyWord *baseAddr = (PolyWord*)obj;
for (POLYUNSIGNED i = 0; i < length; i++)
{
PolyWord someAddr = baseAddr[i];
if (someAddr.IsDataPtr())
{
LocalMemSpace *someSpace = gMem.LocalSpaceForAddress(someAddr.AsAddress());
if (someSpace != 0)
{
PolyObject *someObj = someAddr.AsObjPtr();
// If this is a weak object the SOME value may refer to an unreferenced
// ref. If so we have to set this entry to NONE. For safety we also
// set the contents of the SOME to TAGGED(0).
ASSERT(someObj->Length() == 1 && someObj->IsWordObject()); // Should be a SOME node.
PolyWord refAddress = someObj->Get(0);
LocalMemSpace *space = gMem.LocalSpaceForAddress(refAddress.AsAddress());
if (space != 0)
// If the ref is permanent it's always there.
{
POLYUNSIGNED new_bitno = space->wordNo(refAddress.AsStackAddr());
if (! space->bitmap.TestBit(new_bitno))
{
// It wasn't marked so it's otherwise unreferenced.
baseAddr[i] = TAGGED(0); // Set it to NONE.
someObj->Set(0, TAGGED(0)); // For safety.
convertedWeak = true;
}
}
}
}
}
}
// This is called via ScanAddressesInRegion to process the permanent mutables. It is
// also called from ScanObjectAddress to process root addresses.
// It processes all the addresses reachable from the object.
void RecursiveScan::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
if (OBJ_IS_BYTE_OBJECT(lengthWord))
{
Completed(obj);
return;
}
while (true)
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
// Get the length and base address. N.B. If this is a code segment
// these will be side-effected by GetConstSegmentForCode.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
PolyWord *baseAddr = (PolyWord*)obj;
if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// It's better to process the whole code object in one go.
ScanAddress::ScanAddressesInObject(obj, lengthWord);
length = 0; // Finished
}
ASSERT(! OBJ_IS_BYTE_OBJECT(lengthWord)); // Check - remove this later
// else it's a normal object,
// If there are only two addresses in this cell that need to be
// followed we follow them immediately and treat this cell as done.
// If there are more than two we push the address of this cell on
// the stack, follow the first address and then rescan it. That way
// list cells are processed once only but we don't overflow the
// stack by pushing all the addresses in a very large vector.
PolyWord *endWord = baseAddr + length;
PolyObject *firstWord = 0;
PolyObject *secondWord = 0;
while (baseAddr != endWord)
{
PolyWord wordAt = *baseAddr;
if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0))
{
// Normal address. We can have words of all zeros at least in the
// situation where we have a partially constructed code segment where
// the constants at the end of the code have not yet been filled in.
if (TestForScan(baseAddr)) // Test value at baseAddr (may side-effect it)
{
PolyObject *wObj = (*baseAddr).AsObjPtr();
if (wObj->IsByteObject())
{
// Can do this now - don't need to push it
MarkAsScanning(wObj);
Completed(wObj);
}
else if (firstWord == 0)
{
firstWord = wObj;
// We mark the word immediately. We can have
// two words in an object that are the same
// and we don't want to process it again.
MarkAsScanning(firstWord);
}
else if (secondWord == 0)
secondWord = wObj;
else break; // More than two words.
}
}
else if (wordAt.IsCodePtr())
{
// If we're processing the constant area of a code segment this could
// be a code address.
PolyObject *oldObject = ObjCodePtrToPtr(wordAt.AsCodePtr());
// Calculate the byte offset of this value within the code object.
POLYUNSIGNED offset = wordAt.AsCodePtr() - (byte*)oldObject;
wordAt = oldObject;
bool test = TestForScan(&wordAt);
// TestForScan may side-effect the word.
PolyObject *newObject = wordAt.AsObjPtr();
wordAt = PolyWord::FromCodePtr((byte*)newObject + offset);
if (wordAt != *baseAddr)
*baseAddr = wordAt;
if (test)
{
if (firstWord == 0)
{
firstWord = newObject;
MarkAsScanning(firstWord);
}
else if (secondWord == 0)
secondWord = newObject;
else break;
}
}
baseAddr++;
}
if (baseAddr == endWord)
{
// We have done everything except possibly firstWord and secondWord.
Completed(obj);
if (secondWord != 0)
{
MarkAsScanning(secondWord);
// Put this on the stack. If this is a list node we will be
// pushing the tail.
PushToStack(secondWord);
}
}
else // Put this back on the stack while we process the first word
PushToStack(obj);
if (firstWord != 0)
// Process it immediately.
obj = firstWord;
else if (StackIsEmpty())
return;
else
obj = PopFromStack();
lengthWord = obj->LengthWord();
}
}
// General purpose object processor, Processes all the addresses in an object.
// Handles the various kinds of object that may contain addresses.
void ScanAddress::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
do
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
if (OBJ_IS_BYTE_OBJECT(lengthWord))
return; /* Nothing more to do */
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
PolyWord *baseAddr = (PolyWord*)obj;
if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// Scan constants within the code.
machineDependent->ScanConstantsWithinCode(obj, obj, length, this);
// Skip to the constants and get ready to scan them.
obj->GetConstSegmentForCode(length, baseAddr, length);
} // else it's a normal object,
PolyWord *endWord = baseAddr + length;
// We want to minimise the actual recursion we perform so we try to
// use tail recursion if we can. We first scan from the end and
// remove any words that don't need recursion.
POLYUNSIGNED lastLengthWord = 0;
while (endWord != baseAddr)
{
PolyWord wordAt = endWord[-1];
if (IS_INT(wordAt) || wordAt == PolyWord::FromUnsigned(0))
endWord--; // Don't need to look at this.
else if ((lastLengthWord = ScanAddressAt(endWord-1)) != 0)
// We need to process this one
break;
else endWord--; // We're not interested in this.
}
if (endWord == baseAddr)
return; // We've done everything.
// There is at least one word that needs to be processed, the
// one at endWord-1.
// Now process from the beginning forward to see if there are
// any words before this that need to be handled. This way we are more
// likely to handle the head of a list by recursion and the
// tail by looping (tail recursion).
while (baseAddr < endWord-1)
{
PolyWord wordAt = *baseAddr;
if (IS_INT(wordAt) || wordAt == PolyWord::FromUnsigned(0))
baseAddr++; // Don't need to look at this.
else
{
POLYUNSIGNED lengthWord = ScanAddressAt(baseAddr);
if (lengthWord != 0)
{
wordAt = *baseAddr; // Reload because it may have been side-effected
// We really have to process this recursively.
if (wordAt.IsCodePtr())
ScanAddressesInObject(ObjCodePtrToPtr(wordAt.AsCodePtr()), lengthWord);
else
ScanAddressesInObject(wordAt.AsObjPtr(), lengthWord);
baseAddr++;
}
else baseAddr++;
}
}
// Finally process the last word we found that has to be processed.
// Do this by looping rather than recursion.
PolyWord wordAt = *baseAddr; // Last word to do.
// This must be an address
if (wordAt.IsCodePtr())
obj = ObjCodePtrToPtr(wordAt.AsCodePtr());
else
obj = wordAt.AsObjPtr();
lengthWord = lastLengthWord;
} while(1);
}
// This is called via ScanAddressesInRegion to process the permanent mutables. It is
// also called from ScanObjectAddress to process root addresses.
// It processes all the addresses reachable from the object.
void MTGCProcessMarkPointers::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord)
{
if (OBJ_IS_BYTE_OBJECT(lengthWord))
return;
while (true)
{
ASSERT (OBJ_IS_LENGTH(lengthWord));
// Get the length and base address. N.B. If this is a code segment
// these will be side-effected by GetConstSegmentForCode.
POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord);
if (OBJ_IS_WEAKREF_OBJECT(lengthWord))
{
// Special case.
ASSERT(OBJ_IS_MUTABLE_OBJECT(lengthWord)); // Should be a mutable.
ASSERT(OBJ_IS_WORD_OBJECT(lengthWord)); // Should be a plain object.
// We need to mark the "SOME" values in this object but we don't mark
// the references contained within the "SOME".
PolyWord *baseAddr = (PolyWord*)obj;
// Mark every word but ignore the result.
for (POLYUNSIGNED i = 0; i < length; i++)
(void)MarkAndTestForScan(baseAddr+i);
// We've finished with this.
length = 0;
}
else if (OBJ_IS_CODE_OBJECT(lengthWord))
{
// It's better to process the whole code object in one go.
ScanAddress::ScanAddressesInObject(obj, lengthWord);
length = 0; // Finished
}
// else it's a normal object,
// If there are only two addresses in this cell that need to be
// followed we follow them immediately and treat this cell as done.
// If there are more than two we push the address of this cell on
// the stack, follow the first address and then rescan it. That way
// list cells are processed once only but we don't overflow the
// stack by pushing all the addresses in a very large vector.
PolyWord *baseAddr = (PolyWord*)obj;
PolyWord *endWord = baseAddr + length;
PolyObject *firstWord = 0;
PolyObject *secondWord = 0;
PolyWord *restartAddr = 0;
if (obj == largeObjectCache[locPtr].base)
{
baseAddr = largeObjectCache[locPtr].current;
ASSERT(baseAddr > (PolyWord*)obj && baseAddr < ((PolyWord*)obj)+length);
if (locPtr == 0) locPtr = LARGECACHE_SIZE-1; else locPtr--;
}
while (baseAddr != endWord)
{
PolyWord wordAt = *baseAddr;
if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0))
{
// Normal address. We can have words of all zeros at least in the
// situation where we have a partially constructed code segment where
// the constants at the end of the code have not yet been filled in.
if (TestForScan(baseAddr))
{
if (firstWord == 0)
firstWord = baseAddr->AsObjPtr();
else if (secondWord == 0)
{
// If we need to rescan because there are three or more words to do
// this is the place we need to restart (or the start of the cell if it's
// small).
restartAddr = baseAddr;
secondWord = baseAddr->AsObjPtr();
}
else break; // More than two words.
}
}
else if (wordAt.IsCodePtr())
{
// If we're processing the constant area of a code segment this could
// be a code address.
// Check that this is actually an address. If we have had a bad pointer
// earlier we may treat some length fields as values.
ASSERT(gMem.SpaceForAddress(wordAt.AsCodePtr()) != 0);
PolyObject *oldObject = ObjCodePtrToPtr(wordAt.AsCodePtr());
// Calculate the byte offset of this value within the code object.
POLYUNSIGNED offset = wordAt.AsCodePtr() - (byte*)oldObject;
wordAt = oldObject;
bool test = TestForScan(&wordAt);
// If we've changed it because we had a left-over forwarding pointer
// we need to update the original.
PolyObject *newObject = wordAt.AsObjPtr();
wordAt = PolyWord::FromCodePtr((byte*)newObject + offset);
if (wordAt != *baseAddr)
*baseAddr = wordAt;
if (test)
{
if (firstWord == 0)
firstWord = newObject;
else if (secondWord == 0)
{
restartAddr = baseAddr;
secondWord = newObject;
}
else break;
}
}
baseAddr++;
}
if (baseAddr != endWord)
// Put this back on the stack while we process the first word
PushToStack(obj, length < largeObjectSize ? 0 : restartAddr, length);
else if (secondWord != 0)
{
// Mark it now because we will process it.
secondWord->SetLengthWord(secondWord->LengthWord() | _OBJ_GC_MARK);
// Put this on the stack. If this is a list node we will be
// pushing the tail.
PushToStack(secondWord);
}
if (firstWord != 0)
{
// Mark it and process it immediately.
firstWord->SetLengthWord(firstWord->LengthWord() | _OBJ_GC_MARK);
obj = firstWord;
}
else if (msp == 0)
{
markStack[msp] = 0; // Really finished
return;
}
else
{
// Clear the item above the top. This really is finished.
if (msp < MARK_STACK_SIZE) markStack[msp] = 0;
// Pop the item from the stack but don't overwrite it yet.
// This allows another thread to steal it if there really
// is nothing else to do. This is only really important
// for large objects.
obj = markStack[--msp]; // Pop something.
}
lengthWord = obj->LengthWord();
}
}
// Copy objects from the source space into an earlier space or up within the
// current space.
static void copyAllData(GCTaskId *id, void * /*arg1*/, void * /*arg2*/)
{
LocalMemSpace *mutableDest = 0, *immutableDest = 0;
for (std::vector<LocalMemSpace*>::reverse_iterator i = gMem.lSpaces.rbegin(); i != gMem.lSpaces.rend(); i++)
{
LocalMemSpace *src = *i;
if (src->spaceOwner == 0)
{
PLocker lock(©Lock);
if (src->spaceOwner == 0)
src->spaceOwner = id;
else continue;
}
else if (src->spaceOwner != id)
continue;
if (debugOptions & DEBUG_GC_ENHANCED)
Log("GC: Copy: copying area %p (thread %p) %s \n", src, id, src->spaceTypeString());
// We start at fullGCLowerLimit which is the lowest marked object in the heap
// N.B. It's essential that the first set bit at or above this corresponds
// to the length word of a real object.
uintptr_t bitno = src->wordNo(src->fullGCLowerLimit);
// Set the limit to the top so we won't rescan this. That can
// only happen if copying takes a very short time and the same
// thread runs multiple tasks.
src->fullGCLowerLimit = src->top;
// src->highest is the bit position that corresponds to the top of
// generation we're copying.
uintptr_t highest = src->wordNo(src->top);
for (;;)
{
if (bitno >= highest) break;
/* SPF version; Invariant: 0 < highest - bitno */
bitno += src->bitmap.CountZeroBits(bitno, highest - bitno);
if (bitno >= highest) break;
/* first set bit corresponds to the length word */
PolyWord *old = src->wordAddr(bitno); /* Old object address */
PolyObject *obj = (PolyObject*)(old+1);
POLYUNSIGNED L = obj->LengthWord();
ASSERT (OBJ_IS_LENGTH(L));
POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L) + 1 ;/* Length of allocation (including length word) */
bitno += n;
// Find a mutable space for the mutable objects and an immutable space for
// the immutables. We copy objects into earlier spaces or within its own
// space but we don't copy an object to a later space. This avoids the
// risk of copying an object multiple times. Previously this copied objects
// into later spaces but that doesn't work well if we have converted old
// saved state segments into local areas. It's much better to delete them
// if possible.
bool isMutable = OBJ_IS_MUTABLE_OBJECT(L);
LocalMemSpace *destSpace = isMutable || immutableDest == 0 ? mutableDest : immutableDest;
PolyWord *newp = FindFreeAndAllocate(destSpace, (src == destSpace) ? bitno : 0, n);
if (newp == 0 && src != destSpace)
{
// See if we can find a different space.
// N.B. FindNextSpace side-effects mutableDest/immutableDest to give the next space.
if (FindNextSpace(src, isMutable ? &mutableDest : &immutableDest, isMutable, id))
{
bitno -= n; // Redo this object
continue;
}
// else just leave it
}
if (newp == 0) /* no room */
{
// We're not going to move this object
// Update src->upperAllocPtr, so the old object doesn't get trampled.
if (old < src->upperAllocPtr)
src->upperAllocPtr = old;
// Previously this continued compressing to try to make space available
// on the next GC. Normally full GCs are infrequent so the chances are
// that at the next GC other data will have been freed. Just stop at
// this point.
// However if we're compressing a mutable area and there is immutable
// data in it we should move those out because the mutable area is scanned
// on every partial GC.
if (! src->isMutable || src->i_marked == 0)
break;
}
else
{
PolyObject *destAddress = (PolyObject*)(newp+1);
obj->SetForwardingPtr(destAddress);
CopyObjectToNewAddress(obj, destAddress, L);
if (debugOptions & DEBUG_GC_DETAIL)
Log("GC: Copy: %p %lu %u -> %p\n", obj, OBJ_OBJECT_LENGTH(L),
GetTypeBits(L), destAddress);
}
}
if (mutableDest == src)
mutableDest = 0;
if (immutableDest == src)
immutableDest = 0;
}
}
// Process one level of the word data.
// N.B. The length words are updated without any locking. This is safe
// because all length words are initially chain entries and a chain entry
// can be replaced by another chain entry, a forwarding pointer or a normal
// length word. Forwarding pointers and normal length words are only ever
// set once. There is a small chance that we could lose some sharing as a
// result of a race condition if a thread defers an object because it
// contains a pointer with a chain entry and later sees an otherwise
// equal object where another thread has replaced the chain with a
// normal address, adds it to the list for immediate processing and
// so never compares the two.
void SortVector::wordDataTask(GCTaskId*, void *a, void *)
{
SortVector *s = (SortVector*)a;
// Partition the objects between those that have pointers to objects that are
// still to be processed and those that have been processed.
if (s->baseObject.objList == ENDOFLIST)
return;
PolyObject *h = s->baseObject.objList;
s->baseObject.objList = ENDOFLIST;
s->baseObject.objCount = 0;
POLYUNSIGNED words = OBJ_OBJECT_LENGTH(s->lengthWord);
s->carryOver = 0;
for (unsigned i = 0; i < 256; i++)
{
// Clear the entries in the hash table but not the sharing count.
s->processObjects[i].objList = ENDOFLIST;
s->processObjects[i].objCount = 0;
}
while (h != ENDOFLIST)
{
PolyObject *next = h->GetForwardingPtr();
bool deferred = false;
for (POLYUNSIGNED i = 0; i < words; i++)
{
PolyWord w = h->Get(i);
if (w.IsDataPtr())
{
PolyObject *p = w.AsObjPtr();
objectState state = getObjectState(p);
if (state == FORWARDED)
{
// Update the addresses of objects that have been merged
h->Set(i, p->GetForwardingPtr());
s->carryOver++;
break;
}
else if (state == CHAINED)
{
// If it is still to be shared leave it
deferred = true;
break; // from the loop
}
}
}
if (deferred)
{
// We can't do it yet: add it back to the list
h->SetForwardingPtr(s->baseObject.objList);
s->baseObject.objList = h;
s->baseObject.objCount++;
}
else
{
// Add it to the hash table.
unsigned char hash = 0;
for (POLYUNSIGNED i = 0; i < words*sizeof(PolyWord); i++)
hash += h->AsBytePtr()[i];
h->SetForwardingPtr(s->processObjects[hash].objList);
s->processObjects[hash].objList = h;
s->processObjects[hash].objCount++;
}
h = next;
}
s->SortData();
}
// Quicksort the list to detect cells with the same content. These are made
// to share and removed from further sorting.
void SortVector::sortList(PolyObject *head, POLYUNSIGNED nItems, POLYUNSIGNED &shareCount)
{
while (nItems > 2)
{
size_t bytesToCompare = OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord);
PolyObject *median = head;
head = head->GetForwardingPtr();
median->SetLengthWord(lengthWord);
PolyObject *left = ENDOFLIST, *right = ENDOFLIST;
POLYUNSIGNED leftCount = 0, rightCount = 0;
while (head != ENDOFLIST)
{
PolyObject *next = head->GetForwardingPtr();
int res = memcmp(median, head, bytesToCompare);
if (res == 0)
{
// Equal - they can share
shareWith(head, median);
shareCount++;
}
else if (res < 0)
{
head->SetForwardingPtr(left);
left = head;
leftCount++;
}
else
{
head->SetForwardingPtr(right);
right = head;
rightCount++;
}
head = next;
}
// We can now drop the median and anything that shares with it.
// Process the smaller partition recursively and the larger by
// tail recursion.
if (leftCount < rightCount)
{
sortList(left, leftCount, shareCount);
head = right;
nItems = rightCount;
}
else
{
sortList(right, rightCount, shareCount);
head = left;
nItems = leftCount;
}
}
if (nItems == 1)
head->SetLengthWord(lengthWord);
else if (nItems == 2)
{
PolyObject *next = head->GetForwardingPtr();
head->SetLengthWord(lengthWord);
if (memcmp(head, next, OBJ_OBJECT_LENGTH(lengthWord)*sizeof(PolyWord)) == 0)
{
shareWith(next, head);
shareCount++;
}
else next->SetLengthWord(lengthWord);
}
}