/* enqueue src at the _end_ of the receiver's queue */
virtual void enqueue(MM_HeapRegionQueue *srcAsPQ)
{
MM_LockingHeapRegionQueue* src = MM_LockingHeapRegionQueue::asLockingHeapRegionQueue(srcAsPQ);
if (NULL == src->_head) { /* Nothing to move - single read needs no lock */
return;
}
lock();
src->lock();
/* Remove from src */
MM_HeapRegionDescriptorSegregated *front = src->_head;
MM_HeapRegionDescriptorSegregated *back = src->_tail;
uintptr_t srcLength = src->_length;
src->_head = NULL;
src->_tail = NULL;
src->_length = 0;
/* Add to back of self */
front->setPrev(_tail); /* OK even if _tail is NULL */
if (_tail == NULL) {
_head = front;
} else {
_tail->setNext(front);
}
_tail = back;
_length += srcLength;
src->unlock();
unlock();
}
/* This method is moved here so that AC can see the large full page and cache it */
uintptr_t *
MM_AllocationContextSegregated::allocateLarge(MM_EnvironmentBase *env, uintptr_t sizeInBytesRequired)
{
uintptr_t neededRegions = _regionPool->divideUpRegion(sizeInBytesRequired);
MM_HeapRegionDescriptorSegregated *region = NULL;
uintptr_t excess = 0;
while (region == NULL && excess < MAX_UINT) {
region = _regionPool->allocateFromRegionPool(env, neededRegions, OMR_SIZECLASSES_LARGE, excess);
excess = (2 * excess) + 1;
}
uintptr_t *result = (region == NULL) ? NULL : (uintptr_t *)region->getLowAddress();
/* Flush the large page right away. */
if (region != NULL) {
/* cache the large full region in AC */
_perContextLargeFullRegions->enqueue(region);
/* reset ACL counts */
region->getMemoryPoolACL()->resetCounts();
}
return result;
}
virtual void
push(MM_FreeHeapRegionList *srcAsFPL)
{
MM_LockingFreeHeapRegionList* src = MM_LockingFreeHeapRegionList::asLockingFreeHeapRegionList(srcAsFPL);
if (src->_head == NULL) { /* Nothing to move - single read needs no lock */
return;
}
lock();
src->lock();
/* Remove from src */
MM_HeapRegionDescriptorSegregated *front = src->_head;
MM_HeapRegionDescriptorSegregated *back = src->_tail;
uintptr_t srcLength = src->_length;
src->_head = NULL;
src->_tail = NULL;
src->_length = 0;
/* Add to front of self */
back->setNext(_head); /* OK even if _head is NULL */
if (_head == NULL) {
_tail = back;
} else {
_head->setPrev(back);
}
_head = front;
_length += srcLength;
src->unlock();
unlock();
}
MM_HeapRegionDescriptorSegregated *dequeueInternal()
{
MM_HeapRegionDescriptorSegregated *result = _head;
if (_head != NULL) {
_length--;
_head = result->getNext();
result->setNext(NULL);
if (NULL == _head) {
_tail = NULL;
} else {
_head->setPrev(NULL);
}
}
return result;
}
void
MM_SweepSchemeSegregated::incrementalSweepLarge(MM_EnvironmentBase *env)
{
/* Sweep through large objects. */
MM_RegionPoolSegregated *regionPool = _memoryPool->getRegionPool();
MM_HeapRegionQueue *largeSweepRegions = regionPool->getLargeSweepRegions();
MM_HeapRegionQueue *largeFullRegions = regionPool->getLargeFullRegions();
MM_HeapRegionDescriptorSegregated *currentRegion;
while ((currentRegion = largeSweepRegions->dequeue()) != NULL) {
sweepRegion(env, currentRegion);
if (currentRegion->getMemoryPoolACL()->getFreeCount() == 0) {
largeFullRegions->enqueue(currentRegion);
} else {
currentRegion->emptyRegionReturned(env);
regionPool->addFreeRegion(env, currentRegion);
}
yieldFromSweep(env);
}
}
bool
MM_AllocationContextSegregated::tryAllocateFromRegionPool(MM_EnvironmentBase *env, uintptr_t sizeClass)
{
MM_HeapRegionDescriptorSegregated *region = _regionPool->allocateFromRegionPool(env, 1, sizeClass, MAX_UINT);
bool result = false;
if(NULL != region) {
/* cache the small full region in AC */
_perContextSmallFullRegions[sizeClass]->enqueue(region);
region->formatFresh(env, sizeClass, region->getLowAddress());
/* A store barrier is required here since the initialization of the new region needs to write-back before
* we make it reachable via _smallRegions (_smallRegions is accessed from outside the lock which covers this write)
*/
MM_AtomicOperations::storeSync();
_smallRegions[sizeClass] = region;
result = true;
}
return result;
}
void enqueueInternal(MM_HeapRegionDescriptorSegregated *region)
{
assert1(NULL == region->getNext() && NULL == region->getPrev());
if (NULL == _head) {
_head = _tail = region;
} else {
_tail->setNext(region);
region->setPrev(_tail);
_tail = region;
}
_length++;
}
/*
* This method should be used with care. In particular, it is wrong to detach from a freelist
* while iterating over it unless the detach stops further iteration.
*/
void
detachInternal(MM_HeapRegionDescriptorSegregated *cur)
{
_length--;
MM_HeapRegionDescriptorSegregated *prev = cur->getPrev();
MM_HeapRegionDescriptorSegregated *next = cur->getNext();
if (prev != NULL) {
Assert_MM_true(prev->getNext() == cur);
prev->setNext(next);
} else {
Assert_MM_true(cur == _head);
}
if (next != NULL) {
Assert_MM_true(next->getPrev() == cur);
next->setPrev(prev);
} else {
Assert_MM_true(cur == _tail);
}
cur->setPrev(NULL);
cur->setNext(NULL);
if (_head == cur) {
_head = next;
}
if (_tail == cur) {
_tail = prev;
}
}
void
MM_SweepSchemeSegregated::incrementalSweepArraylet(MM_EnvironmentBase *env)
{
uintptr_t arrayletsPerRegion = env->getExtensions()->arrayletsPerRegion;
MM_RegionPoolSegregated *regionPool = _memoryPool->getRegionPool();
MM_HeapRegionQueue *arrayletSweepRegions = regionPool->getArrayletSweepRegions();
MM_HeapRegionQueue *arrayletAvailableRegions = regionPool->getArrayletAvailableRegions();
MM_HeapRegionDescriptorSegregated *currentRegion;
while ((currentRegion = arrayletSweepRegions->dequeue()) != NULL) {
sweepRegion(env, currentRegion);
if (currentRegion->getMemoryPoolACL()->getFreeCount() != arrayletsPerRegion) {
arrayletAvailableRegions->enqueue(currentRegion);
} else {
currentRegion->emptyRegionReturned(env);
regionPool->addFreeRegion(env, currentRegion);
}
yieldFromSweep(env);
}
}
void
pushInternal(MM_HeapRegionDescriptorSegregated *region)
{
Assert_MM_true(NULL == region->getNext() && NULL == region->getPrev());
_length++;
if (NULL == _head) {
_head = region;
_tail = region;
} else {
_head->setPrev(region);
region->setNext(_head);
_head = region;
}
}
void
MM_SweepSchemeSegregated::incrementalSweepSmall(MM_EnvironmentBase *env)
{
MM_GCExtensionsBase *ext = env->getExtensions();
bool shouldUpdateOccupancy = ext->nonDeterministicSweep;
MM_RegionPoolSegregated *regionPool = _memoryPool->getRegionPool();
uintptr_t splitIndex = env->getSlaveID() % (regionPool->getSplitAvailableListSplitCount());
/*
* Iterate through the regions so that each region is processed exactly once.
* Interleaved sweeping: free up some number of regions of all sizeClasses right away
* so that application have somewhere to allocate from and minimize the usage of
* non deterministic sweeps.
* Any marked objects are unmarked.
* If a region holds a marked object, then the region is kept active;
* if a region contains no marked objects, then it can be returned to a free list.
*/
MM_SizeClasses *sizeClasses = ext->defaultSizeClasses;
while (regionPool->getCurrentTotalCountOfSweepRegions()) {
for (uintptr_t sizeClass = OMR_SIZECLASSES_MIN_SMALL; sizeClass <= OMR_SIZECLASSES_MAX_SMALL; sizeClass++) {
while (regionPool->getCurrentCountOfSweepRegions(sizeClass)) {
float yetToComplete = (float)regionPool->getCurrentCountOfSweepRegions(sizeClass) / regionPool->getInitialCountOfSweepRegions(sizeClass);
float totalYetToComplete = (float)regionPool->getCurrentTotalCountOfSweepRegions() / regionPool->getInitialTotalCountOfSweepRegions();
if (yetToComplete < totalYetToComplete) {
break;
}
MM_HeapRegionQueue *sweepList = regionPool->getSmallSweepRegions(sizeClass);
MM_HeapRegionDescriptorSegregated *currentRegion;
uintptr_t numCells = sizeClasses->getNumCells(sizeClass);
uintptr_t sweepSmallRegionsPerIteration = calcSweepSmallRegionsPerIteration(numCells);
uintptr_t yieldSlackTime = resetSweepSmallRegionCount(env, sweepSmallRegionsPerIteration);
uintptr_t actualSweepRegions;
if ((actualSweepRegions = sweepList->dequeue(env->getRegionWorkList(), sweepSmallRegionsPerIteration)) > 0) {
regionPool->decrementCurrentCountOfSweepRegions(sizeClass, actualSweepRegions);
regionPool->decrementCurrentTotalCountOfSweepRegions(actualSweepRegions);
uintptr_t freedRegions = 0, processedRegions = 0;
MM_HeapRegionQueue *fullList = env->getRegionLocalFull();
while ((currentRegion = env->getRegionWorkList()->dequeue()) != NULL) {
sweepRegion(env, currentRegion);
if (currentRegion->getMemoryPoolACL()->getFreeCount() < numCells) {
uintptr_t occupancy = (currentRegion->getMemoryPoolACL()->getMarkCount() * 100) / numCells;
/* Maintain average occupancy needed for nondeterministic sweep heuristic */
if (shouldUpdateOccupancy) {
regionPool->updateOccupancy(sizeClass, occupancy);
}
if (currentRegion->getMemoryPoolACL()->getMarkCount() == numCells) {
/* Return full regions to full list */
fullList->enqueue(currentRegion);
} else {
regionPool->enqueueAvailable(currentRegion, sizeClass, occupancy, splitIndex);
}
} else {
currentRegion->emptyRegionReturned(env);
currentRegion->setFree(1);
env->getRegionLocalFree()->enqueue(currentRegion);
freedRegions++;
}
processedRegions++;
if (updateSweepSmallRegionCount()) {
yieldFromSweep(env, yieldSlackTime);
}
}
regionPool->addSingleFree(env, env->getRegionLocalFree());
regionPool->getSmallFullRegions(sizeClass)->enqueue(fullList);
yieldFromSweep(env, yieldSlackTime);
}
} /* end of while(currentTotalCountOfSweepRegions); */
}
}
}
void
MM_SweepSchemeSegregated::incrementalCoalesceFreeRegions(MM_EnvironmentBase *env)
{
resetCoalesceFreeRegionCount(env);
MM_GCExtensionsBase *ext = env->getExtensions();
MM_HeapRegionManager *regionManager = ext->heap->getHeapRegionManager();
uintptr_t regionCount = regionManager->getTableRegionCount();
MM_RegionPoolSegregated *regionPool = _memoryPool->getRegionPool();
MM_FreeHeapRegionList *coalesceFreeList = regionPool->getCoalesceFreeList();
uintptr_t yieldSlackTime = resetCoalesceFreeRegionCount(env);
yieldFromSweep(env, yieldSlackTime);
coalesceFreeList->push(regionPool->getSingleFreeList());
coalesceFreeList->push(regionPool->getMultiFreeList());
MM_HeapRegionDescriptorSegregated *coalescing = NULL;
MM_HeapRegionDescriptorSegregated *currentRegion = NULL;
for (uintptr_t i=0; i< regionCount; ) {
currentRegion = (MM_HeapRegionDescriptorSegregated *)regionManager->mapRegionTableIndexToDescriptor(i);
uintptr_t range = currentRegion->getRange();
i += range;
bool shouldYield = updateCoalesceFreeRegionCount(range);
bool shouldClose = shouldYield || (i >= regionCount);
if (currentRegion->isFree()) {
coalesceFreeList->detach(currentRegion);
bool joined = (range < MAX_REGION_COALESCE) && (coalescing != NULL && coalescing->joinFreeRangeInit(currentRegion));
if (joined) {
currentRegion = NULL;
} else {
shouldClose = true;
}
} else {
currentRegion = NULL;
}
if (shouldClose && coalescing != NULL) {
coalescing->joinFreeRangeComplete();
regionPool->addFreeRegion(env, coalescing, true);
coalescing = NULL;
}
if (shouldYield) {
if (currentRegion != NULL) {
regionPool->addFreeRegion(env, currentRegion, true);
currentRegion = NULL;
}
yieldFromSweep(env, yieldSlackTime);
} else {
if (coalescing == NULL) {
coalescing = currentRegion;
}
}
}
if (currentRegion != NULL) {
regionPool->addFreeRegion(env, currentRegion, true);
currentRegion = NULL;
}
yieldFromSweep(env);
}
/*
* Pre allocate a list of cells, the amount to pre-allocate is retrieved from the env's allocation interface.
* @return the carved off first cell in the list
*/
uintptr_t *
MM_AllocationContextSegregated::preAllocateSmall(MM_EnvironmentBase *env, uintptr_t sizeInBytesRequired)
{
MM_SizeClasses *sizeClasses = env->getExtensions()->defaultSizeClasses;
uintptr_t sizeClass = sizeClasses->getSizeClassSmall(sizeInBytesRequired);
uintptr_t sweepCount = 0;
uint64_t sweepStartTime = 0;
bool done = false;
uintptr_t *result = NULL;
/* BEN TODO 1429: The object allocation interface base class should define all API used by this method such that casting would be unnecessary. */
MM_SegregatedAllocationInterface* segregatedAllocationInterface = (MM_SegregatedAllocationInterface*)env->_objectAllocationInterface;
uintptr_t replenishSize = segregatedAllocationInterface->getReplenishSize(env, sizeInBytesRequired);
uintptr_t preAllocatedBytes = 0;
while (!done) {
/* If we have a region, attempt to replenish the ACL's cache */
MM_HeapRegionDescriptorSegregated *region = _smallRegions[sizeClass];
if (NULL != region) {
MM_MemoryPoolAggregatedCellList *memoryPoolACL = region->getMemoryPoolACL();
uintptr_t* cellList = memoryPoolACL->preAllocateCells(env, sizeClasses->getCellSize(sizeClass), replenishSize, &preAllocatedBytes);
if (NULL != cellList) {
Assert_MM_true(preAllocatedBytes > 0);
if (shouldPreMarkSmallCells(env)) {
_markingScheme->preMarkSmallCells(env, region, cellList, preAllocatedBytes);
}
segregatedAllocationInterface->replenishCache(env, sizeInBytesRequired, cellList, preAllocatedBytes);
result = (uintptr_t *) segregatedAllocationInterface->allocateFromCache(env, sizeInBytesRequired);
done = true;
}
}
smallAllocationLock();
/* Either we did not have a region or we failed to preAllocate from the ACL. Retry if this is no
* longer true */
region = _smallRegions[sizeClass];
if ((NULL == region) || !region->getMemoryPoolACL()->hasCell()) {
/* This may cause the start of a GC */
signalSmallRegionDepleted(env, sizeClass);
flushSmall(env, sizeClass);
/* Attempt to get a region of this size class which may already have some allocated cells */
if (!tryAllocateRegionFromSmallSizeClass(env, sizeClass)) {
/* Attempt to get a region by sweeping */
if (!trySweepAndAllocateRegionFromSmallSizeClass(env, sizeClass, &sweepCount, &sweepStartTime)) {
/* Attempt to get an unused region */
if (!tryAllocateFromRegionPool(env, sizeClass)) {
/* Really out of regions */
done = true;
}
}
}
}
smallAllocationUnlock();
}
return result;
}
