static void SetBitmaps(LocalMemSpace *space, PolyWord *pt, PolyWord *top) { while (pt < top) { PolyObject *obj = (PolyObject*)++pt; // If it has been copied by a minor collection skip it if (obj->ContainsForwardingPtr()) { obj = FollowForwarding(obj); ASSERT(obj->ContainsNormalLengthWord()); pt += obj->Length(); } else { POLYUNSIGNED L = obj->LengthWord(); POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L); if (L & _OBJ_GC_MARK) { obj->SetLengthWord(L & ~(_OBJ_GC_MARK)); POLYUNSIGNED bitno = space->wordNo(pt); space->bitmap.SetBits(bitno - 1, n + 1); if (OBJ_IS_MUTABLE_OBJECT(L)) space->m_marked += n + 1; else space->i_marked += n + 1; if ((PolyWord*)obj <= space->fullGCLowerLimit) space->fullGCLowerLimit = (PolyWord*)obj-1; if (OBJ_IS_WEAKREF_OBJECT(L)) { // Add this to the limits for the containing area. PolyWord *baseAddr = (PolyWord*)obj; PolyWord *startAddr = baseAddr-1; // Must point AT length word. PolyWord *endObject = baseAddr + n; if (startAddr < space->lowestWeak) space->lowestWeak = startAddr; if (endObject > space->highestWeak) space->highestWeak = endObject; } } pt += n; } } }
// Deal with weak objects void MTGCCheckWeakRef::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED L) { if (! OBJ_IS_WEAKREF_OBJECT(L)) return; ASSERT(OBJ_IS_MUTABLE_OBJECT(L)); // Should be a mutable. ASSERT(OBJ_IS_WORD_OBJECT(L)); // Should be a plain object. // See if any of the SOME objects contain unreferenced refs. POLYUNSIGNED length = OBJ_OBJECT_LENGTH(L); PolyWord *baseAddr = (PolyWord*)obj; for (POLYUNSIGNED i = 0; i < length; i++) { PolyWord someAddr = baseAddr[i]; if (someAddr.IsDataPtr()) { LocalMemSpace *someSpace = gMem.LocalSpaceForAddress(someAddr.AsAddress()); if (someSpace != 0) { PolyObject *someObj = someAddr.AsObjPtr(); // If this is a weak object the SOME value may refer to an unreferenced // ref. If so we have to set this entry to NONE. For safety we also // set the contents of the SOME to TAGGED(0). ASSERT(someObj->Length() == 1 && someObj->IsWordObject()); // Should be a SOME node. PolyWord refAddress = someObj->Get(0); LocalMemSpace *space = gMem.LocalSpaceForAddress(refAddress.AsAddress()); if (space != 0) // If the ref is permanent it's always there. { POLYUNSIGNED new_bitno = space->wordNo(refAddress.AsStackAddr()); if (! space->bitmap.TestBit(new_bitno)) { // It wasn't marked so it's otherwise unreferenced. baseAddr[i] = TAGGED(0); // Set it to NONE. someObj->Set(0, TAGGED(0)); // For safety. convertedWeak = true; } } } } } }
void GetSharing::Completed(PolyObject *obj) { // We mustn't include cells in the permanent area. // We scan the permanent mutable areas for local addresses // but we mustn't add the cells themselves. Normally they // will be mutable so would be ignored but cells that have been // locked will now be immutable. The test in TestForScan is bypassed // by ScanAddressesInRegion. PolyWord *lengthWord = ((PolyWord*)obj) - 1; if (gMem.LocalSpaceForAddress(lengthWord) == 0) return; POLYUNSIGNED L = obj->LengthWord(); // We have tables for word objects and byte objects // We chain entries together using the length word so it // is important that we only do this for objects that // have no other bits in the header, such as the sign bit. if ((L & _OBJ_PRIVATE_FLAGS_MASK) == 0) { POLYUNSIGNED length = obj->Length(); if (length < NUM_WORD_VECTORS) wordVectors[length].AddToVector(obj, length); else largeWordCount++; wordAdded++; } else if ((L & _OBJ_PRIVATE_FLAGS_MASK) == _OBJ_BYTE_OBJ) { POLYUNSIGNED length = obj->Length(); if (length < NUM_BYTE_VECTORS) byteVectors[length].AddToVector(obj, length); else largeByteCount++; byteAdded++; } else if (! OBJ_IS_CODE_OBJECT(L) && ! OBJ_IS_MUTABLE_OBJECT(L)) excludedCount++; // Code and mutables can't be shared - see what could be // TODO: We don't attempt to share closure cells in 32-in-64. }
// This is called via ScanAddressesInRegion to process the permanent mutables. It is // also called from ScanObjectAddress to process root addresses. // It processes all the addresses reachable from the object. void MTGCProcessMarkPointers::ScanAddressesInObject(PolyObject *obj, POLYUNSIGNED lengthWord) { if (OBJ_IS_BYTE_OBJECT(lengthWord)) return; while (true) { ASSERT (OBJ_IS_LENGTH(lengthWord)); // Get the length and base address. N.B. If this is a code segment // these will be side-effected by GetConstSegmentForCode. POLYUNSIGNED length = OBJ_OBJECT_LENGTH(lengthWord); if (OBJ_IS_WEAKREF_OBJECT(lengthWord)) { // Special case. ASSERT(OBJ_IS_MUTABLE_OBJECT(lengthWord)); // Should be a mutable. ASSERT(OBJ_IS_WORD_OBJECT(lengthWord)); // Should be a plain object. // We need to mark the "SOME" values in this object but we don't mark // the references contained within the "SOME". PolyWord *baseAddr = (PolyWord*)obj; // Mark every word but ignore the result. for (POLYUNSIGNED i = 0; i < length; i++) (void)MarkAndTestForScan(baseAddr+i); // We've finished with this. length = 0; } else if (OBJ_IS_CODE_OBJECT(lengthWord)) { // It's better to process the whole code object in one go. ScanAddress::ScanAddressesInObject(obj, lengthWord); length = 0; // Finished } // else it's a normal object, // If there are only two addresses in this cell that need to be // followed we follow them immediately and treat this cell as done. // If there are more than two we push the address of this cell on // the stack, follow the first address and then rescan it. That way // list cells are processed once only but we don't overflow the // stack by pushing all the addresses in a very large vector. PolyWord *baseAddr = (PolyWord*)obj; PolyWord *endWord = baseAddr + length; PolyObject *firstWord = 0; PolyObject *secondWord = 0; PolyWord *restartAddr = 0; if (obj == largeObjectCache[locPtr].base) { baseAddr = largeObjectCache[locPtr].current; ASSERT(baseAddr > (PolyWord*)obj && baseAddr < ((PolyWord*)obj)+length); if (locPtr == 0) locPtr = LARGECACHE_SIZE-1; else locPtr--; } while (baseAddr != endWord) { PolyWord wordAt = *baseAddr; if (wordAt.IsDataPtr() && wordAt != PolyWord::FromUnsigned(0)) { // Normal address. We can have words of all zeros at least in the // situation where we have a partially constructed code segment where // the constants at the end of the code have not yet been filled in. if (TestForScan(baseAddr)) { if (firstWord == 0) firstWord = baseAddr->AsObjPtr(); else if (secondWord == 0) { // If we need to rescan because there are three or more words to do // this is the place we need to restart (or the start of the cell if it's // small). restartAddr = baseAddr; secondWord = baseAddr->AsObjPtr(); } else break; // More than two words. } } else if (wordAt.IsCodePtr()) { // If we're processing the constant area of a code segment this could // be a code address. // Check that this is actually an address. If we have had a bad pointer // earlier we may treat some length fields as values. ASSERT(gMem.SpaceForAddress(wordAt.AsCodePtr()) != 0); PolyObject *oldObject = ObjCodePtrToPtr(wordAt.AsCodePtr()); // Calculate the byte offset of this value within the code object. POLYUNSIGNED offset = wordAt.AsCodePtr() - (byte*)oldObject; wordAt = oldObject; bool test = TestForScan(&wordAt); // If we've changed it because we had a left-over forwarding pointer // we need to update the original. PolyObject *newObject = wordAt.AsObjPtr(); wordAt = PolyWord::FromCodePtr((byte*)newObject + offset); if (wordAt != *baseAddr) *baseAddr = wordAt; if (test) { if (firstWord == 0) firstWord = newObject; else if (secondWord == 0) { restartAddr = baseAddr; secondWord = newObject; } else break; } } baseAddr++; } if (baseAddr != endWord) // Put this back on the stack while we process the first word PushToStack(obj, length < largeObjectSize ? 0 : restartAddr, length); else if (secondWord != 0) { // Mark it now because we will process it. secondWord->SetLengthWord(secondWord->LengthWord() | _OBJ_GC_MARK); // Put this on the stack. If this is a list node we will be // pushing the tail. PushToStack(secondWord); } if (firstWord != 0) { // Mark it and process it immediately. firstWord->SetLengthWord(firstWord->LengthWord() | _OBJ_GC_MARK); obj = firstWord; } else if (msp == 0) { markStack[msp] = 0; // Really finished return; } else { // Clear the item above the top. This really is finished. if (msp < MARK_STACK_SIZE) markStack[msp] = 0; // Pop the item from the stack but don't overwrite it yet. // This allows another thread to steal it if there really // is nothing else to do. This is only really important // for large objects. obj = markStack[--msp]; // Pop something. } lengthWord = obj->LengthWord(); } }
// Copy objects from the source space into an earlier space or up within the // current space. static void copyAllData(GCTaskId *id, void * /*arg1*/, void * /*arg2*/) { LocalMemSpace *mutableDest = 0, *immutableDest = 0; for (std::vector<LocalMemSpace*>::reverse_iterator i = gMem.lSpaces.rbegin(); i != gMem.lSpaces.rend(); i++) { LocalMemSpace *src = *i; if (src->spaceOwner == 0) { PLocker lock(©Lock); if (src->spaceOwner == 0) src->spaceOwner = id; else continue; } else if (src->spaceOwner != id) continue; if (debugOptions & DEBUG_GC_ENHANCED) Log("GC: Copy: copying area %p (thread %p) %s \n", src, id, src->spaceTypeString()); // We start at fullGCLowerLimit which is the lowest marked object in the heap // N.B. It's essential that the first set bit at or above this corresponds // to the length word of a real object. uintptr_t bitno = src->wordNo(src->fullGCLowerLimit); // Set the limit to the top so we won't rescan this. That can // only happen if copying takes a very short time and the same // thread runs multiple tasks. src->fullGCLowerLimit = src->top; // src->highest is the bit position that corresponds to the top of // generation we're copying. uintptr_t highest = src->wordNo(src->top); for (;;) { if (bitno >= highest) break; /* SPF version; Invariant: 0 < highest - bitno */ bitno += src->bitmap.CountZeroBits(bitno, highest - bitno); if (bitno >= highest) break; /* first set bit corresponds to the length word */ PolyWord *old = src->wordAddr(bitno); /* Old object address */ PolyObject *obj = (PolyObject*)(old+1); POLYUNSIGNED L = obj->LengthWord(); ASSERT (OBJ_IS_LENGTH(L)); POLYUNSIGNED n = OBJ_OBJECT_LENGTH(L) + 1 ;/* Length of allocation (including length word) */ bitno += n; // Find a mutable space for the mutable objects and an immutable space for // the immutables. We copy objects into earlier spaces or within its own // space but we don't copy an object to a later space. This avoids the // risk of copying an object multiple times. Previously this copied objects // into later spaces but that doesn't work well if we have converted old // saved state segments into local areas. It's much better to delete them // if possible. bool isMutable = OBJ_IS_MUTABLE_OBJECT(L); LocalMemSpace *destSpace = isMutable || immutableDest == 0 ? mutableDest : immutableDest; PolyWord *newp = FindFreeAndAllocate(destSpace, (src == destSpace) ? bitno : 0, n); if (newp == 0 && src != destSpace) { // See if we can find a different space. // N.B. FindNextSpace side-effects mutableDest/immutableDest to give the next space. if (FindNextSpace(src, isMutable ? &mutableDest : &immutableDest, isMutable, id)) { bitno -= n; // Redo this object continue; } // else just leave it } if (newp == 0) /* no room */ { // We're not going to move this object // Update src->upperAllocPtr, so the old object doesn't get trampled. if (old < src->upperAllocPtr) src->upperAllocPtr = old; // Previously this continued compressing to try to make space available // on the next GC. Normally full GCs are infrequent so the chances are // that at the next GC other data will have been freed. Just stop at // this point. // However if we're compressing a mutable area and there is immutable // data in it we should move those out because the mutable area is scanned // on every partial GC. if (! src->isMutable || src->i_marked == 0) break; } else { PolyObject *destAddress = (PolyObject*)(newp+1); obj->SetForwardingPtr(destAddress); CopyObjectToNewAddress(obj, destAddress, L); if (debugOptions & DEBUG_GC_DETAIL) Log("GC: Copy: %p %lu %u -> %p\n", obj, OBJ_OBJECT_LENGTH(L), GetTypeBits(L), destAddress); } } if (mutableDest == src) mutableDest = 0; if (immutableDest == src) immutableDest = 0; } }