void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other)
{
ASSERT(m_segmentCapacity == other.m_segmentCapacity);
validatePrevious();
other.validatePrevious();
// If other has an entire segment, steal it and return.
if (other.m_topSegment->m_previous) {
ASSERT(other.m_topSegment->m_previous->m_top == m_segmentCapacity);
// First remove a segment from other.
MarkStackSegment* current = other.m_topSegment->m_previous;
other.m_topSegment->m_previous = current->m_previous;
other.m_numberOfPreviousSegments--;
ASSERT(!!other.m_numberOfPreviousSegments == !!other.m_topSegment->m_previous);
// Now add it to this.
current->m_previous = m_topSegment->m_previous;
m_topSegment->m_previous = current;
m_numberOfPreviousSegments++;
validatePrevious();
other.validatePrevious();
return;
}
// Otherwise drain 1/Nth of the shared array where N is the number of
// workers, or Options::minimumNumberOfCellsToKeep, whichever is bigger.
size_t numberOfCellsToSteal = std::max((size_t)Options::minimumNumberOfCellsToKeep, other.size() / Options::numberOfGCMarkers);
while (numberOfCellsToSteal-- > 0 && other.canRemoveLast())
append(other.removeLast());
}
void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other, size_t idleThreadCount)
{
// Try to steal 1 / Nth of the shared array, where N is the number of idle threads.
// To reduce copying costs, we prefer stealing a whole segment over stealing
// individual cells, even if this skews away from our 1 / N target.
validatePrevious();
other.validatePrevious();
// If other has an entire segment, steal it and return.
if (other.m_numberOfSegments > 1) {
// Move the heads of the lists aside. We'll push them back on after.
MarkStackSegment* otherHead = other.m_segments.removeHead();
MarkStackSegment* myHead = m_segments.removeHead();
ASSERT(other.m_segments.head()->m_top == s_segmentCapacity);
m_segments.push(other.m_segments.removeHead());
m_numberOfSegments++;
other.m_numberOfSegments--;
m_segments.push(myHead);
other.m_segments.push(otherHead);
validatePrevious();
other.validatePrevious();
return;
}
size_t numberOfCellsToSteal = (other.size() + idleThreadCount - 1) / idleThreadCount; // Round up to steal 1 / 1.
while (numberOfCellsToSteal-- > 0 && other.canRemoveLast())
append(other.removeLast());
}
void MarkStackArray::stealSomeCellsFrom(MarkStackArray& other, size_t idleThreadCount)
{
// Try to steal 1 / Nth of the shared array, where N is the number of idle threads.
// To reduce copying costs, we prefer stealing a whole segment over stealing
// individual cells, even if this skews away from our 1 / N target.
ASSERT(m_segmentCapacity == other.m_segmentCapacity);
validatePrevious();
other.validatePrevious();
// If other has an entire segment, steal it and return.
if (other.m_topSegment->m_previous) {
ASSERT(other.m_topSegment->m_previous->m_top == m_segmentCapacity);
// First remove a segment from other.
MarkStackSegment* current = other.m_topSegment->m_previous;
other.m_topSegment->m_previous = current->m_previous;
other.m_numberOfPreviousSegments--;
ASSERT(!!other.m_numberOfPreviousSegments == !!other.m_topSegment->m_previous);
// Now add it to this.
current->m_previous = m_topSegment->m_previous;
m_topSegment->m_previous = current;
m_numberOfPreviousSegments++;
validatePrevious();
other.validatePrevious();
return;
}
size_t numberOfCellsToSteal = (other.size() + idleThreadCount - 1) / idleThreadCount; // Round up to steal 1 / 1.
while (numberOfCellsToSteal-- > 0 && other.canRemoveLast())
append(other.removeLast());
}
bool MarkStackArray::refill()
{
validatePrevious();
if (top())
return true;
m_blockAllocator.deallocate(MarkStackSegment::destroy(m_segments.removeHead()));
ASSERT(m_numberOfSegments > 1);
m_numberOfSegments--;
setTopForFullSegment();
validatePrevious();
return true;
}
bool MarkStackArray::refill()
{
validatePrevious();
if (top())
return true;
MarkStackSegment* toFree = m_topSegment;
MarkStackSegment* previous = m_topSegment->m_previous;
if (!previous)
return false;
ASSERT(m_numberOfPreviousSegments);
m_numberOfPreviousSegments--;
m_topSegment = previous;
m_allocator.release(toFree);
setTopForFullSegment();
validatePrevious();
return true;
}
bool MarkStackArray::donateSomeCellsTo(MarkStackArray& other)
{
ASSERT(m_segmentCapacity == other.m_segmentCapacity);
validatePrevious();
other.validatePrevious();
// Fast check: see if the other mark stack already has enough segments.
if (other.m_numberOfPreviousSegments + 1 >= Options::maximumNumberOfSharedSegments)
return false;
size_t numberOfCellsToKeep = Options::minimumNumberOfCellsToKeep;
ASSERT(m_top > numberOfCellsToKeep || m_topSegment->m_previous);
// Looks like we should donate! Give the other mark stack all of our
// previous segments, and then top it off.
MarkStackSegment* previous = m_topSegment->m_previous;
while (previous) {
ASSERT(m_numberOfPreviousSegments);
MarkStackSegment* current = previous;
previous = current->m_previous;
current->m_previous = other.m_topSegment->m_previous;
other.m_topSegment->m_previous = current;
m_numberOfPreviousSegments--;
other.m_numberOfPreviousSegments++;
}
ASSERT(!m_numberOfPreviousSegments);
m_topSegment->m_previous = 0;
validatePrevious();
other.validatePrevious();
// Now top off. We want to keep at a minimum numberOfCellsToKeep, but if
// we really have a lot of work, we give up half.
if (m_top > numberOfCellsToKeep * 2)
numberOfCellsToKeep = m_top / 2;
while (m_top > numberOfCellsToKeep)
other.append(removeLast());
return true;
}
void MarkStackArray::donateSomeCellsTo(MarkStackArray& other)
{
// Try to donate about 1 / 2 of our cells. To reduce copying costs,
// we prefer donating whole segments over donating individual cells,
// even if this skews away from our 1 / 2 target.
size_t segmentsToDonate = m_numberOfSegments / 2; // If we only have one segment (our head) we don't donate any segments.
if (!segmentsToDonate) {
size_t cellsToDonate = m_top / 2; // Round down to donate 0 / 1 cells.
while (cellsToDonate--) {
ASSERT(m_top);
other.append(removeLast());
}
return;
}
validatePrevious();
other.validatePrevious();
// Remove our head and the head of the other list before we start moving segments around.
// We'll add them back on once we're done donating.
MarkStackSegment* myHead = m_segments.removeHead();
MarkStackSegment* otherHead = other.m_segments.removeHead();
while (segmentsToDonate--) {
MarkStackSegment* current = m_segments.removeHead();
ASSERT(current);
ASSERT(m_numberOfSegments > 1);
other.m_segments.push(current);
m_numberOfSegments--;
other.m_numberOfSegments++;
}
// Put the original heads back in their places.
m_segments.push(myHead);
other.m_segments.push(otherHead);
validatePrevious();
other.validatePrevious();
}
void MarkStackArray::donateSomeCellsTo(MarkStackArray& other)
{
// Try to donate about 1 / 2 of our cells. To reduce copying costs,
// we prefer donating whole segments over donating individual cells,
// even if this skews away from our 1 / 2 target.
ASSERT(m_segmentCapacity == other.m_segmentCapacity);
size_t segmentsToDonate = (m_numberOfPreviousSegments + 2 - 1) / 2; // Round up to donate 1 / 1 previous segments.
if (!segmentsToDonate) {
size_t cellsToDonate = m_top / 2; // Round down to donate 0 / 1 cells.
while (cellsToDonate--) {
ASSERT(m_top);
other.append(removeLast());
}
return;
}
validatePrevious();
other.validatePrevious();
MarkStackSegment* previous = m_topSegment->m_previous;
while (segmentsToDonate--) {
ASSERT(previous);
ASSERT(m_numberOfPreviousSegments);
MarkStackSegment* current = previous;
previous = current->m_previous;
current->m_previous = other.m_topSegment->m_previous;
other.m_topSegment->m_previous = current;
m_numberOfPreviousSegments--;
other.m_numberOfPreviousSegments++;
}
m_topSegment->m_previous = previous;
validatePrevious();
other.validatePrevious();
}
void MarkStackArray::expand()
{
ASSERT(m_topSegment->m_top == m_segmentCapacity);
m_numberOfPreviousSegments++;
MarkStackSegment* nextSegment = m_allocator.allocate();
#if !ASSERT_DISABLED
nextSegment->m_top = 0;
#endif
nextSegment->m_previous = m_topSegment;
m_topSegment = nextSegment;
setTopForEmptySegment();
validatePrevious();
}
void MarkStackArray::expand()
{
ASSERT(m_segments.head()->m_top == s_segmentCapacity);
MarkStackSegment* nextSegment = MarkStackSegment::create(m_blockAllocator.allocate<MarkStackSegment>());
m_numberOfSegments++;
#if !ASSERT_DISABLED
nextSegment->m_top = 0;
#endif
m_segments.push(nextSegment);
setTopForEmptySegment();
validatePrevious();
}