void
MM_CompactStats::merge(MM_CompactStats *statsToMerge)
{
_movedObjects += statsToMerge->_movedObjects;
_movedBytes += statsToMerge->_movedBytes;
_fixupObjects += statsToMerge->_fixupObjects;
/* merging time intervals is a little different than just creating a total since the sum of two time intervals, for our uses, is their union (as opposed to the sum of two time spans, which is their sum) */
_setupStartTime = (0 == _setupStartTime) ? statsToMerge->_setupStartTime : OMR_MIN(_setupStartTime, statsToMerge->_setupStartTime);
_setupEndTime = OMR_MAX(_setupEndTime, statsToMerge->_setupEndTime);
_moveStartTime = (0 == _moveStartTime) ? statsToMerge->_moveStartTime : OMR_MIN(_moveStartTime, statsToMerge->_moveStartTime);
_moveEndTime = OMR_MAX(_moveEndTime, statsToMerge->_moveEndTime);
_fixupStartTime = (0 == _fixupStartTime) ? statsToMerge->_fixupStartTime : OMR_MIN(_fixupStartTime, statsToMerge->_fixupStartTime);
_fixupEndTime = OMR_MAX(_fixupEndTime, statsToMerge->_fixupEndTime);
_rootFixupStartTime = (0 == _rootFixupStartTime) ? statsToMerge->_rootFixupStartTime : OMR_MIN(_rootFixupStartTime, statsToMerge->_rootFixupStartTime);
_rootFixupEndTime = OMR_MAX(_rootFixupEndTime, statsToMerge->_rootFixupEndTime);
};
uintptr_t
MM_ParallelSweepScheme::performSamplingCalculations(MM_ParallelSweepChunk *sweepChunk, uintptr_t* markMapCurrent, uintptr_t* heapSlotFreeCurrent)
{
const uintptr_t minimumFreeEntrySize = sweepChunk->memoryPool->getMinimumFreeEntrySize();
uintptr_t darkMatter = 0;
/* this word has objects in it. Sample them for dark matter */
MM_HeapMapWordIterator markedObjectIterator(*markMapCurrent, heapSlotFreeCurrent);
/* Hole at the beginning of the sample is not considered, since we do not know
* if that's part of a preceding object or part of hole.
*/
omrobjectptr_t prevObject = markedObjectIterator.nextObject();
Assert_MM_true(NULL != prevObject);
uintptr_t prevObjectSize = _extensions->objectModel.getConsumedSizeInBytesWithHeader(prevObject);
omrobjectptr_t object = NULL;
while (NULL != (object = markedObjectIterator.nextObject())) {
uintptr_t holeSize = (uintptr_t)object - ((uintptr_t)prevObject + prevObjectSize);
Assert_MM_true(holeSize < minimumFreeEntrySize);
darkMatter += holeSize;
prevObject = object;
prevObjectSize = _extensions->objectModel.getConsumedSizeInBytesWithHeader(prevObject);
}
/* find the trailing dark matter */
uintptr_t * endOfPrevObject = (uintptr_t*)((uintptr_t)prevObject + prevObjectSize);
uintptr_t * startSearchAt = (uintptr_t*)MM_Math::roundToFloor(J9MODRON_HEAP_SLOTS_PER_MARK_SLOT * sizeof(uintptr_t), (uintptr_t)endOfPrevObject);
uintptr_t * endSearchAt = (uintptr_t*)MM_Math::roundToCeiling(J9MODRON_HEAP_SLOTS_PER_MARK_SLOT * sizeof(uintptr_t), (uintptr_t)endOfPrevObject + minimumFreeEntrySize);
startSearchAt = OMR_MAX(startSearchAt, heapSlotFreeCurrent + J9MODRON_HEAP_SLOTS_PER_MARK_SLOT);
endSearchAt = OMR_MIN(endSearchAt, (uintptr_t*)sweepChunk->chunkTop);
if (startSearchAt < endSearchAt) {
while ( startSearchAt < endSearchAt ) {
MM_HeapMapWordIterator nextMarkedObjectIterator(_currentMarkMap, startSearchAt);
omrobjectptr_t nextObject = nextMarkedObjectIterator.nextObject();
if (NULL != nextObject) {
uintptr_t holeSize = (uintptr_t)nextObject - (uintptr_t)endOfPrevObject;
if (holeSize < minimumFreeEntrySize) {
darkMatter += holeSize;
}
break;
}
startSearchAt += J9MODRON_HEAP_SLOTS_PER_MARK_SLOT;
}
} else if (endSearchAt > endOfPrevObject) {
darkMatter += (uintptr_t)endSearchAt - (uintptr_t)endOfPrevObject;
}
return darkMatter;
}
void
MM_ConfigurationStandard::initializeGCThreadCount(MM_EnvironmentBase* env)
{
MM_Configuration::initializeGCThreadCount(env);
#if defined(OMR_GC_CONCURRENT_SCAVENGER)
MM_GCExtensionsBase* extensions = env->getExtensions();
/* If not explicitly set, concurrent phase of CS runs with approx 1/4 the thread count (relative to STW phases thread count */
if (!extensions->concurrentScavengerBackgroundThreadsForced) {
extensions->concurrentScavengerBackgroundThreads = OMR_MAX(1, (extensions->gcThreadCount + 1) / 4);
} else if (extensions->concurrentScavengerBackgroundThreads > extensions->gcThreadCount) {
extensions->concurrentScavengerBackgroundThreads = extensions->gcThreadCount;
}
#endif
}
/**
* Count the number of "active" cards in a given chunk of card table.
* As a chunk may contain holes (segmented heap) we use the _cleaningRanges
* structure to work out how many cards there are within a defined range
* of cards; "chunkStart" thru "chunkEnd".
*
* @param chunkStart First card in the chunk of cards to be processed
* @param chunkEnd Last card in the chunk
*
* @return Number of active cards in the the chunk
*/
uintptr_t
MM_ConcurrentCardTableForWC::countCardsInRange(MM_EnvironmentBase *env, Card *chunkStart, Card *chunkEnd)
{
uintptr_t cardsInRange = 0;
CleaningRange *cleaningRange = _cleaningRanges;
/* Find cleaning range that corresponds to start of card table range to be counted */
while (cleaningRange < _lastCleaningRange && cleaningRange->topCard < chunkStart) {
cleaningRange++;
}
/* Count all cards until we get to end of all cleaning ranges or end of
* chunk whose cards we are counting
*/
while (cleaningRange < _lastCleaningRange && cleaningRange->baseCard < chunkEnd) {
Card *first = OMR_MAX(chunkStart, cleaningRange->baseCard);
Card *last = OMR_MIN(chunkEnd,cleaningRange->topCard);
/* Add cards in this cleaning range into total cards to be cleaned */
cardsInRange += (uintptr_t)(last - first);
cleaningRange++;
}
return cardsInRange;
}
bool
MM_NUMAManager::recacheNUMASupport(MM_EnvironmentBase *env)
{
bool result = true;
if (NULL != _activeNodes) {
env->getForge()->free(_activeNodes);
_activeNodes = NULL;
_activeNodeCount = 0;
}
if (NULL != _affinityLeaders) {
env->getForge()->free(_affinityLeaders);
_affinityLeaders = NULL;
_affinityLeaderCount = 0;
}
if (NULL != _freeProcessorPoolNodes) {
env->getForge()->free(_freeProcessorPoolNodes);
_freeProcessorPoolNodes = NULL;
_freeProcessorPoolNodeCount = 0;
}
_maximumNodeNumber = 0;
uintptr_t nodeCount = 0;
OMRPORT_ACCESS_FROM_OMRPORT(env->getPortLibrary());
if (_physicalNumaEnabled) {
intptr_t detailResult = omrvmem_numa_get_node_details(NULL, &nodeCount);
if (0 != detailResult) {
/* something went wrong in the underlying call so ignore any NUMA count data we might have received */
nodeCount = 0;
}
} else {
nodeCount = _simulatedNodeCount;
}
if (0 != nodeCount) {
/* we want to support NUMA either via the machine's physical NUMA or our simulated (aka "purely logical") NUMA */
uintptr_t nodeArraySize = sizeof(J9MemoryNodeDetail) * nodeCount;
_activeNodes = (J9MemoryNodeDetail *)env->getForge()->allocate(nodeArraySize, OMR::GC::AllocationCategory::FIXED, OMR_GET_CALLSITE());
if (NULL == _activeNodes) {
result = false;
} else {
memset(_activeNodes, 0x0, nodeArraySize);
_activeNodeCount = nodeCount;
if (_physicalNumaEnabled) {
/* we want REAL NUMA so ask the underlying system for node details */
intptr_t detailResult = omrvmem_numa_get_node_details(_activeNodes, &_activeNodeCount);
Assert_MM_true(0 == detailResult);
Assert_MM_true(_activeNodeCount == nodeCount);
} else {
/* we want to use "fake" NUMA so synthesize the array data */
for (uintptr_t i = 0; i < nodeCount; i++) {
_activeNodes[i].j9NodeNumber = i + 1;
_activeNodes[i].memoryPolicy = J9NUMA_PREFERRED;
/* NOTE: if we ever start counting these resources to determine GC thread counts, this will have to be something other than "1" */
_activeNodes[i].computationalResourcesAvailable = 1;
}
}
/* Sorting the array in j9NodeNumber ascending order.
* This sorting is not really required, but gives us ability to make some stronger assertions later in the code
*/
J9_SORT(_activeNodes, _activeNodeCount, sizeof(J9MemoryNodeDetail), compareNodeNumberFunc);
/* now that we have the total list of active nodes, identify the affinity leaders */
uintptr_t preferredWithCPU = 0;
uintptr_t allowedWithCPU = 0;
/* if "preferred" nodes exist, we will use them as the leaders and fall back to "allowed" if there aren't any so count both and choose them after the loop */
for (uintptr_t activeNodeIndex = 0; activeNodeIndex < _activeNodeCount; activeNodeIndex++) {
uintptr_t cpu = _activeNodes[activeNodeIndex].computationalResourcesAvailable;
if (0 != cpu) {
J9MemoryState policy = _activeNodes[activeNodeIndex].memoryPolicy;
if (J9NUMA_PREFERRED == policy) {
preferredWithCPU += 1;
} else if (J9NUMA_ALLOWED == policy) {
allowedWithCPU += 1;
} else {
/* nodes with CPUs but DENIED bindings go into the free processor pool */
_freeProcessorPoolNodeCount += 1;
}
}
/* since we are walking all the nodes, this is also the time to update the maximum node number */
_maximumNodeNumber = OMR_MAX(_maximumNodeNumber, _activeNodes[activeNodeIndex].j9NodeNumber);
}
J9MemoryState policyType = J9NUMA_PREFERRED;
_affinityLeaderCount = preferredWithCPU;
if (0 == _affinityLeaderCount) {
_affinityLeaderCount = allowedWithCPU;
policyType = J9NUMA_ALLOWED;
}
/* Affinity Leader array allocation and construction */
if (0 != _affinityLeaderCount) {
uintptr_t affinityLeaderArraySize = sizeof(J9MemoryNodeDetail) * _affinityLeaderCount;
_affinityLeaders = (J9MemoryNodeDetail *)env->getForge()->allocate(affinityLeaderArraySize, OMR::GC::AllocationCategory::FIXED, OMR_GET_CALLSITE());
if (NULL == _affinityLeaders) {
result = false;
} else {
memset(_affinityLeaders, 0x0, affinityLeaderArraySize);
uintptr_t nextIndex = 0;
for (uintptr_t activeNodeIndex = 0; activeNodeIndex < _activeNodeCount; activeNodeIndex++) {
if ((0 != _activeNodes[activeNodeIndex].computationalResourcesAvailable) && (policyType == _activeNodes[activeNodeIndex].memoryPolicy)) {
Assert_MM_true(nextIndex < _affinityLeaderCount);
_affinityLeaders[nextIndex] = _activeNodes[activeNodeIndex];
nextIndex += 1;
}
}
Assert_MM_true(nextIndex == _affinityLeaderCount);
}
}
/* Free Processor Pool array allocation and construction */
if (0 != _freeProcessorPoolNodeCount) {
/* allocate a free processor pool */
uintptr_t processorPoolArraySize = sizeof(J9MemoryNodeDetail) * _freeProcessorPoolNodeCount;
_freeProcessorPoolNodes = (J9MemoryNodeDetail *)env->getForge()->allocate(processorPoolArraySize, OMR::GC::AllocationCategory::FIXED, OMR_GET_CALLSITE());
if (NULL == _freeProcessorPoolNodes) {
result = false;
} else {
memset(_freeProcessorPoolNodes, 0x0, processorPoolArraySize);
uintptr_t nextIndex = 0;
for (uintptr_t activeNodeIndex = 0; activeNodeIndex < _activeNodeCount; activeNodeIndex++) {
if ((0 != _activeNodes[activeNodeIndex].computationalResourcesAvailable) && (J9NUMA_DENIED == _activeNodes[activeNodeIndex].memoryPolicy)) {
Assert_MM_true(nextIndex < _freeProcessorPoolNodeCount);
_freeProcessorPoolNodes[nextIndex] = _activeNodes[activeNodeIndex];
nextIndex += 1;
}
}
Assert_MM_true(nextIndex == _freeProcessorPoolNodeCount);
}
}
}
}
return result;
}
/**
* Create the type of memorypool that corresponds to this Standard configuration.
* In a generational mode this corresponds to the "old" area.
* This is abstracted away into a superclass to avoid code duplication between all the Standard configurations.
*
* @return the appropriate memoryPool
*/
MM_MemoryPool*
MM_ConfigurationStandard::createMemoryPool(MM_EnvironmentBase* env, bool appendCollectorLargeAllocateStats)
{
MM_GCExtensionsBase* extensions = env->getExtensions();
uintptr_t minimumFreeEntrySize = extensions->tlhMinimumSize;
bool doSplit = 1 < extensions->splitFreeListSplitAmount;
bool doHybrid = extensions->enableHybridMemoryPool;
#if defined(OMR_GC_CONCURRENT_SWEEP)
if (extensions->concurrentSweep) {
doSplit = false;
/* TODO: enable processLargeAllocateStats in concurrentSweep case, currently can not collect correct FreeEntry Stats in concurrentSweep case*/
extensions->processLargeAllocateStats = false;
extensions->estimateFragmentation = NO_ESTIMATE_FRAGMENTATION;
}
#endif /* OMR_GC_CONCURRENT_SWEEP */
if ((UDATA_MAX == extensions->largeObjectAllocationProfilingVeryLargeObjectThreshold) && extensions->processLargeAllocateStats) {
extensions->largeObjectAllocationProfilingVeryLargeObjectThreshold = OMR_MAX(10*1024*1024, extensions->memoryMax/100);
}
/* Create Sweep Pool Manager for Tenure */
if (doSplit) {
if (doHybrid) {
if (!createSweepPoolManagerHybrid(env)) {
return NULL;
}
}
else {
if (!createSweepPoolManagerSplitAddressOrderedList(env)) {
return NULL;
}
}
if (extensions->largeObjectArea) {
/* need non-split version for loa as well */
if (!createSweepPoolManagerAddressOrderedList(env)) {
return NULL;
}
}
} else {
if (!createSweepPoolManagerAddressOrderedList(env)) {
return NULL;
}
}
if (extensions->largeObjectArea) {
MM_MemoryPoolAddressOrderedListBase* memoryPoolLargeObjects = NULL;
MM_MemoryPoolAddressOrderedListBase* memoryPoolSmallObjects = NULL;
/* create memory pools for SOA and LOA */
if (doSplit) {
memoryPoolSmallObjects = MM_MemoryPoolSplitAddressOrderedList::newInstance(env, minimumFreeEntrySize, extensions->splitFreeListSplitAmount, "SOA");
} else {
memoryPoolSmallObjects = MM_MemoryPoolAddressOrderedList::newInstance(env, minimumFreeEntrySize, "SOA");
}
if (NULL == memoryPoolSmallObjects) {
return NULL;
}
memoryPoolLargeObjects = MM_MemoryPoolAddressOrderedList::newInstance(env, extensions->largeObjectMinimumSize, "LOA");
if (NULL == memoryPoolLargeObjects) {
memoryPoolSmallObjects->kill(env);
return NULL;
}
if (appendCollectorLargeAllocateStats) {
memoryPoolLargeObjects->appendCollectorLargeAllocateStats();
memoryPoolSmallObjects->appendCollectorLargeAllocateStats();
}
if (!extensions->freeEntrySizeClassStatsSimulated.initialize(env, (uint16_t)extensions->largeObjectAllocationProfilingTopK, extensions->freeMemoryProfileMaxSizeClasses, extensions->largeObjectAllocationProfilingVeryLargeObjectThreshold, 1, true)) {
memoryPoolSmallObjects->kill(env);
memoryPoolLargeObjects->kill(env);
return NULL;
}
return MM_MemoryPoolLargeObjects::newInstance(env, memoryPoolLargeObjects, memoryPoolSmallObjects);
} else {
MM_MemoryPool* memoryPool = NULL;
if (doSplit) {
memoryPool = MM_MemoryPoolSplitAddressOrderedList::newInstance(env, minimumFreeEntrySize, extensions->splitFreeListSplitAmount, "Tenure");
} else {
memoryPool = MM_MemoryPoolAddressOrderedList::newInstance(env, minimumFreeEntrySize, "Tenure");
}
if (NULL == memoryPool) {
return NULL;
}
if (appendCollectorLargeAllocateStats) {
memoryPool->appendCollectorLargeAllocateStats();
}
if (!extensions->freeEntrySizeClassStatsSimulated.initialize(env, (uint16_t)extensions->largeObjectAllocationProfilingTopK, extensions->freeMemoryProfileMaxSizeClasses, extensions->largeObjectAllocationProfilingVeryLargeObjectThreshold, 1, true)) {
memoryPool->kill(env);
return NULL;
}
return memoryPool;
}
}
bool
MM_MemoryPoolSplitAddressOrderedListBase::initialize(MM_EnvironmentBase* env)
{
MM_GCExtensionsBase* extensions = env->getExtensions();
if (!MM_MemoryPool::initialize(env)) {
return false;
}
/*
* Create Sweep Pool State for MPAOL
*/
MM_Collector* globalCollector = _extensions->getGlobalCollector();
Assert_MM_true(NULL != globalCollector);
_sweepPoolState = static_cast<MM_SweepPoolState*>(globalCollector->createSweepPoolState(env, this));
if (NULL == _sweepPoolState) {
return false;
}
/*
* Get Sweep Pool Manager for MPAOL
* For platforms it doesn't required still be NULL
*/
_sweepPoolManager = extensions->sweepPoolManagerSmallObjectArea;
_currentThreadFreeList = (uintptr_t*)extensions->getForge()->allocate(sizeof(uintptr_t) * _heapFreeListCount, OMR::GC::AllocationCategory::FIXED, OMR_GET_CALLSITE());
if (NULL == _currentThreadFreeList) {
return false;
} else {
for (uintptr_t i = 0; i < _heapFreeListCount; ++i) {
_currentThreadFreeList[i] = 0;
}
}
_heapFreeLists = (J9ModronFreeList*)extensions->getForge()->allocate(sizeof(J9ModronFreeList) * _heapFreeListCountExtended, OMR::GC::AllocationCategory::FIXED, OMR_GET_CALLSITE());
if (NULL == _heapFreeLists) {
return false;
} else {
for (uintptr_t i = 0; i < _heapFreeListCountExtended; ++i) {
_heapFreeLists[i] = J9ModronFreeList();
if (!_heapFreeLists[i].initialize(env)) {
return false;
}
}
_referenceHeapFreeList = &(_heapFreeLists[0]._freeList);
}
#if defined(OMR_GC_MODRON_SCAVENGER)
/* this memoryPool can be used by scavenger, maximum tlh size should be max(_extensions->tlhMaximumSize, _extensions->scavengerScanCacheMaximumSize) */
uintptr_t tlhMaximumSize = OMR_MAX(_extensions->tlhMaximumSize, _extensions->scavengerScanCacheMaximumSize);
#else /* OMR_GC_MODRON_SCAVENGER */
uintptr_t tlhMaximumSize = _extensions->tlhMaximumSize;
#endif /* OMR_GC_MODRON_SCAVENGER */
/* set multiple factor = 2 for doubling _maxVeryLargeEntrySizes to avoid run out of _veryLargeEntryPool (minus count during decrement) */
_largeObjectAllocateStats = MM_LargeObjectAllocateStats::newInstance(env, (uint16_t)extensions->largeObjectAllocationProfilingTopK, extensions->largeObjectAllocationProfilingThreshold, extensions->largeObjectAllocationProfilingVeryLargeObjectThreshold, (float)extensions->largeObjectAllocationProfilingSizeClassRatio / (float)100.0,
_extensions->heap->getMaximumMemorySize(), tlhMaximumSize + _minimumFreeEntrySize, _extensions->tlhMinimumSize, 2);
if (NULL == _largeObjectAllocateStats) {
return false;
}
/* If we ever subclass MM_LargeObjectAllocateStats we will have to allocate this as an array of pointers to MM_LargeObjectAllocateStats and invoke newInstance instead of just initialize
* Than, we may also need to virtualize initialize() and tearDown().
*/
_largeObjectAllocateStatsForFreeList = (MM_LargeObjectAllocateStats*)extensions->getForge()->allocate(sizeof(MM_LargeObjectAllocateStats) * _heapFreeListCountExtended, OMR::GC::AllocationCategory::FIXED, OMR_GET_CALLSITE());
if (NULL == _largeObjectAllocateStatsForFreeList) {
return false;
} else {
for (uintptr_t i = 0; i < _heapFreeListCountExtended; ++i) {
new (&_largeObjectAllocateStatsForFreeList[i]) MM_LargeObjectAllocateStats();
/* set multiple factor = 2 for doubling _maxVeryLargeEntrySizes to avoid run out of _veryLargeEntryPool (minus count during decrement) */
if (!_largeObjectAllocateStatsForFreeList[i].initialize(env, (uint16_t)extensions->largeObjectAllocationProfilingTopK, extensions->largeObjectAllocationProfilingThreshold, extensions->largeObjectAllocationProfilingVeryLargeObjectThreshold, (float)extensions->largeObjectAllocationProfilingSizeClassRatio / (float)100.0,
_extensions->heap->getMaximumMemorySize(), tlhMaximumSize + _minimumFreeEntrySize, _extensions->tlhMinimumSize, 2)) {
return false;
}
}
}
/* At this moment we do not know who is creator of this pool, so we do not set _largeObjectCollectorAllocateStatsForFreeList yet.
* Tenure SubSpace for Gencon will set _largeObjectCollectorAllocateStatsForFreeList to _largeObjectCollectorAllocateStatsForFreeList (we append collector stats to mutator stats)
* Tenure SubSpace for Flat will leave _largeObjectCollectorAllocateStatsForFreeList at NULL (no interest in collector stats)
* We do not even need _largeObjectCollectorAllocateStats here (as we do in plain MPAOL) - we'll just merge whatever _largeObjectCollectorAllocateStatsForFreeList contatins (appended Collector stats or not)
*/
if (!_resetLock.initialize(env, &extensions->lnrlOptions, "MM_MemoryPoolSplitAddressOrderedList:_resetLock")) {
return false;
}
return true;
}
