LargeMemoryBlock *ExtMemoryPool::mallocLargeObject(size_t allocationSize)
{
#if __TBB_MALLOC_LOCACHE_STAT
AtomicIncrement(mallocCalls);
AtomicAdd(memAllocKB, allocationSize/1024);
#endif
LargeMemoryBlock* lmb = loc.get(allocationSize);
if (!lmb) {
BackRefIdx backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/true);
if (backRefIdx.isInvalid())
return NULL;
// unalignedSize is set in getLargeBlock
lmb = backend.getLargeBlock(allocationSize);
if (!lmb) {
removeBackRef(backRefIdx);
loc.rollbackCacheState(allocationSize);
return NULL;
}
lmb->backRefIdx = backRefIdx;
STAT_increment(getThreadId(), ThreadCommonCounters, allocNewLargeObj);
} else {
#if __TBB_MALLOC_LOCACHE_STAT
AtomicIncrement(cacheHits);
AtomicAdd(memHitKB, allocationSize/1024);
#endif
}
return lmb;
}
void ExtMemoryPool::freeLargeObject(void *object)
{
LargeObjectHdr *header = (LargeObjectHdr*)object - 1;
// overwrite backRefIdx to simplify double free detection
header->backRefIdx = BackRefIdx();
if (!loc.put(this, header->memoryBlock)) {
removeBackRef(header->memoryBlock->backRefIdx);
backend.putLargeBlock(header->memoryBlock);
STAT_increment(getThreadId(), ThreadCommonCounters, freeLargeObj);
}
}
bool LargeObjectCache::put(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock)
{
size_t idx = sizeToIdx(largeBlock->unalignedSize);
if (idx<numLargeBlockBins) {
MALLOC_ITT_SYNC_RELEASING(bin+idx);
if (bin[idx].put(extMemPool, largeBlock)) {
STAT_increment(getThreadId(), ThreadCommonCounters, cacheLargeBlk);
return true;
} else
return false;
}
return false;
}
LargeMemoryBlock *LargeObjectCache::get(ExtMemoryPool *extMemPool, size_t size)
{
MALLOC_ASSERT( size%largeBlockCacheStep==0, ASSERT_TEXT );
LargeMemoryBlock *lmb = NULL;
size_t idx = sizeToIdx(size);
if (idx<numLargeBlockBins) {
lmb = bin[idx].get(extMemPool, size);
if (lmb) {
MALLOC_ITT_SYNC_ACQUIRED(bin+idx);
STAT_increment(getThreadId(), ThreadCommonCounters, allocCachedLargeBlk);
}
}
return lmb;
}
LargeMemoryBlock *LargeObjectCacheImpl<Props>::get(uintptr_t currTime, size_t size)
{
MALLOC_ASSERT( size%Props::CacheStep==0, ASSERT_TEXT );
int idx = sizeToIdx(size);
bool setNonEmpty = false;
LargeMemoryBlock *lmb = bin[idx].get(size, currTime, &setNonEmpty);
// Setting to true is possible out of lock. As bitmask is used only for cleanup,
// the lack of consistency is not violating correctness here.
if (setNonEmpty)
bitMask.set(idx, true);
if (lmb) {
MALLOC_ITT_SYNC_ACQUIRED(bin+idx);
STAT_increment(getThreadId(), ThreadCommonCounters, allocCachedLargeObj);
}
return lmb;
}
void *ExtMemoryPool::mallocLargeObject(size_t size, size_t alignment)
{
size_t headersSize = sizeof(LargeMemoryBlock)+sizeof(LargeObjectHdr);
// TODO: take into account that they are already largeObjectAlignment-aligned
size_t allocationSize = alignUp(size+headersSize+alignment, largeBlockCacheStep);
if (allocationSize < size) // allocationSize is wrapped around after alignUp
return NULL;
LargeMemoryBlock* lmb = loc.get(this, allocationSize);
if (!lmb) {
BackRefIdx backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/true);
if (backRefIdx.isInvalid())
return NULL;
// unalignedSize is set in getLargeBlock
lmb = backend.getLargeBlock(allocationSize);
if (!lmb) {
removeBackRef(backRefIdx);
return NULL;
}
lmb->backRefIdx = backRefIdx;
STAT_increment(getThreadId(), ThreadCommonCounters, allocNewLargeObj);
}
void *alignedArea = (void*)alignUp((uintptr_t)lmb+headersSize, alignment);
LargeObjectHdr *header = (LargeObjectHdr*)alignedArea-1;
header->memoryBlock = lmb;
header->backRefIdx = lmb->backRefIdx;
setBackRef(header->backRefIdx, header);
lmb->objectSize = size;
MALLOC_ASSERT( isLargeObject(alignedArea), ASSERT_TEXT );
return alignedArea;
}
LargeMemoryBlock *LargeObjectCacheImpl<Props>::CacheBin::
putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx)
{
int i, num, totalNum;
size_t size = head->unalignedSize;
LargeMemoryBlock *curr, *tail, *toRelease = NULL;
uintptr_t currTime;
// we not kept prev pointers during assigning blocks to bins, set them now
head->prev = NULL;
for (num=1, curr=head; curr->next; num++, curr=curr->next)
curr->next->prev = curr;
tail = curr;
totalNum = num;
{
MallocMutex::scoped_lock scoped_cs(lock);
usedSize -= num*size;
// to keep ordering on list, get time under list lock
currTime = extMemPool->loc.getCurrTimeRange(num);
for (curr=tail, i=0; curr; curr=curr->prev, i++) {
curr->age = currTime+i;
STAT_increment(getThreadId(), ThreadCommonCounters, cacheLargeObj);
}
if (!lastCleanedAge) {
// 1st object of such size was released.
// Not cache it, and remeber when this occurs
// to take into account during cache miss.
lastCleanedAge = tail->age;
toRelease = tail;
tail = tail->prev;
if (tail)
tail->next = NULL;
else
head = NULL;
num--;
}
if (num) {
// add [head;tail] list to cache
tail->next = first;
if (first)
first->prev = tail;
first = head;
if (!last) {
MALLOC_ASSERT(0 == oldest, ASSERT_TEXT);
oldest = tail->age;
last = tail;
}
cachedSize += num*size;
}
/* It's accebtable, if a bin is empty, and we have non-empty in bit mask.
So set true in bitmask without lock.
It's not acceptable, if a bin is non-empty and we have empty in bitmask.
So set false in bitmask under lock. */
// No used object, and nothing in the bin, mark the bin as empty
if (!usedSize && !first)
bitMask->set(idx, false);
}
extMemPool->loc.cleanupCacheIfNeededOnRange(totalNum, currTime);
if (toRelease)
toRelease->prev = toRelease->next = NULL;
return toRelease;
}
