static void TestEntries()
{
const int SZ = 4;
const int ALGN = 4;
size_t size[SZ] = {8, 8000, 9000, 100*1024};
size_t algn[ALGN] = {8, 64, 4*1024, 8*1024*1024};
rml::MemPoolPolicy pol(getGranMem, putGranMem);
currGranularity = 1; // not check granularity in the test
rml::MemoryPool *pool;
pool_create_v1(0, &pol, &pool);
for (int i=0; i<SZ; i++)
for (int j=0; j<ALGN; j++) {
char *p = (char*)pool_aligned_malloc(pool, size[i], algn[j]);
ASSERT(p && 0==((uintptr_t)p & (algn[j]-1)), NULL);
memset(p, j, size[i]);
size_t curr_algn = algn[rand() % ALGN];
size_t curr_sz = size[rand() % SZ];
char *p1 = (char*)pool_aligned_realloc(pool, p, curr_sz, curr_algn);
ASSERT(p1 && 0==((uintptr_t)p1 & (curr_algn-1)), NULL);
ASSERT(memEqual(p1, min(size[i], curr_sz), j), NULL);
memset(p1, j+1, curr_sz);
size_t curr_sz1 = size[rand() % SZ];
char *p2 = (char*)pool_realloc(pool, p1, curr_sz1);
ASSERT(p2, NULL);
ASSERT(memEqual(p2, min(curr_sz1, curr_sz), j+1), NULL);
pool_free(pool, p2);
}
pool_destroy(pool);
}
void TestFixedBufferPool()
{
void *ptrs[7];
rml::MemPoolPolicy pol(fixedBufGetMem, NULL, 0, /*fixedSizePool=*/true,
/*keepMemTillDestroy=*/false);
rml::MemoryPool *pool;
pool_create_v1(0, &pol, &pool);
void *largeObj = pool_malloc(pool, 7*1024*1024);
ASSERT(largeObj, NULL);
pool_free(pool, largeObj);
for (int i=0; i<7; i++) {
ptrs[i] = pool_malloc(pool, 1024*1024);
ASSERT(ptrs[i], NULL);
}
for (int i=0; i<7; i++)
pool_free(pool, ptrs[i]);
largeObj = pool_malloc(pool, 7*1024*1024);
ASSERT(largeObj, NULL);
pool_free(pool, largeObj);
pool_destroy(pool);
}
void operator()( int id ) const {
rml::MemPoolPolicy pol(CrossThreadGetMem, CrossThreadPutMem);
const int objLen = 10*id;
pool_create_v1(id, &pol, &pool[id]);
obj[id] = (char*)pool_malloc(pool[id], objLen);
ASSERT(obj[id], NULL);
memset(obj[id], id, objLen);
{
const size_t lrgSz = 2*16*1024;
void *ptrLarge = pool_malloc(pool[id], lrgSz);
ASSERT(ptrLarge, NULL);
memset(ptrLarge, 1, lrgSz);
// consume all small objects
while (pool_malloc(pool[id], 5*1024))
;
// releasing of large object can give a chance to allocate more
pool_free(pool[id], ptrLarge);
ASSERT(pool_malloc(pool[id], 5*1024), NULL);
}
barrier.wait();
int myPool = number_of_threads-id-1;
for (int i=0; i<10*myPool; i++)
ASSERT(myPool==obj[myPool][i], NULL);
pool_free(pool[myPool], obj[myPool]);
pool_destroy(pool[myPool]);
}
// single pool shared by different threads
void TestSharedPool()
{
rml::MemPoolPolicy pol(getMallocMem, putMallocMem);
rml::MemoryPool *pool;
pool_create_v1(0, &pol, &pool);
void **crossThread = new void*[MaxThread * SharedPoolRun::OBJ_CNT];
void **afterTerm = new void*[MaxThread * SharedPoolRun::OBJ_CNT];
for (int p=MinThread; p<=MaxThread; p++) {
SharedPoolRun::init(p, pool, crossThread, afterTerm);
SharedPoolRun thr;
void *hugeObj = pool_malloc(pool, 10*1024*1024);
ASSERT(hugeObj, NULL);
NativeParallelFor( p, thr );
pool_free(pool, hugeObj);
for (int i=0; i<p*SharedPoolRun::OBJ_CNT; i++)
pool_free(pool, afterTerm[i]);
}
delete []afterTerm;
delete []crossThread;
pool_destroy(pool);
ASSERT(!liveRegions, "Expected all regions were released.");
}
void tbb_initialize(struct memkind *kind)
{
if(!kind || !TBBInitDone) {
log_fatal("Failed to initialize TBB.");
abort();
}
struct MemPoolPolicy policy = {
.pAlloc = raw_alloc,
.pFree = raw_free,
.granularity = GRANULARITY,
.version = 1,
.fixedPool = false,
.keepAllMemory = false,
.reserved = 0
};
pool_create_v1((intptr_t)kind, &policy, &kind->priv);
if (!kind->priv) {
log_fatal("Unable to create TBB memory pool.");
abort();
}
kind->ops->malloc = tbb_pool_malloc;
kind->ops->calloc = tbb_pool_calloc;
kind->ops->posix_memalign = tbb_pool_posix_memalign;
kind->ops->realloc = tbb_pool_realloc;
kind->ops->free = tbb_pool_free;
kind->ops->finalize = tbb_destroy;
kind->ops->malloc_usable_size = tbb_pool_malloc_usable_size;
kind->ops->update_memory_usage_policy = tbb_update_memory_usage_policy;
}
/* test that pools in small space are either usable or not created
(i.e., exception raised) */
void TestSmallFixedSizePool()
{
char *buf;
bool allocated = false;
for (size_t sz = 0; sz < 64*1024; sz = sz? 3*sz : 3) {
buf = (char*)malloc(sz);
#if TBB_USE_EXCEPTIONS
try {
tbb::fixed_pool pool(buf, sz);
/* Check that pool is usable, i.e. such an allocation exists,
that can be fulfilled from the pool. 16B allocation fits in 16KB slabs,
so it requires at least 16KB. Requirement of 9KB allocation is more modest.
*/
allocated = pool.malloc( 16 ) || pool.malloc( 9*1024 );
ASSERT(allocated, "If pool created, it must be useful.");
} catch (std::bad_alloc) {
} catch (...) {
ASSERT(0, "wrong exception type; expected bad_alloc");
}
#else
/* Do not test high-level pool interface because pool ctor emit exception
on creation failure. Instead test same functionality via low-level interface.
TODO: add support for configuration with disabled exceptions to pools.
*/
rml::MemPoolPolicy pol(fixedBufGetMem, NULL, 0, /*fixedSizePool=*/true,
/*keepMemTillDestroy=*/false);
rml::MemoryPool *pool;
FixedPool fixedPool(buf, sz);
rml::MemPoolError ret = pool_create_v1((intptr_t)&fixedPool, &pol, &pool);
if (ret == rml::POOL_OK) {
allocated = pool_malloc(pool, 16) || pool_malloc(pool, 9*1024);
ASSERT(allocated, "If pool created, it must be useful.");
pool_destroy(pool);
} else
ASSERT(ret == rml::NO_MEMORY, "Expected that pool either valid "
"or have no memory to be created");
#endif
free(buf);
}
ASSERT(allocated, "Maximal buf size should be enough to create working fixed_pool");
#if TBB_USE_EXCEPTIONS
try {
tbb::fixed_pool pool(NULL, 10*1024*1024);
ASSERT(0, "Useless allocator with no memory must not be created");
} catch (std::bad_alloc) {
} catch (...) {
ASSERT(0, "wrong exception type; expected bad_alloc");
}
#endif
}
// buffer is too small to pool be created, but must not leak resourses
void TestTooSmallBuffer()
{
poolSpace = new PoolSpace(8*1024);
rml::MemPoolPolicy pol(CrossThreadGetMem, CrossThreadPutMem);
rml::MemoryPool *pool;
pool_create_v1(0, &pol, &pool);
pool_destroy(pool);
ASSERT(!poolSpace[0].regions, "No leaks.");
delete poolSpace;
}
static void TestPoolGranularity()
{
rml::MemPoolPolicy pol(getGranMem, putGranMem);
const size_t grans[] = {4*1024, 2*1024*1024, 6*1024*1024, 10*1024*1024};
for (unsigned i=0; i<sizeof(grans)/sizeof(grans[0]); i++) {
pol.granularity = currGranularity = grans[i];
rml::MemoryPool *pool;
pool_create_v1(0, &pol, &pool);
for (int sz=500*1024; sz<16*1024*1024; sz+=101*1024) {
void *p = pool_malloc(pool, sz);
ASSERT(p, "Can't allocate memory in pool.");
pool_free(pool, p);
}
pool_destroy(pool);
}
}
static void TestPoolKeepTillDestroy()
{
const int ITERS = 50*1024;
void *ptrs[2*ITERS+1];
rml::MemPoolPolicy pol(getMemPolicy, putMemPolicy);
rml::MemoryPool *pool;
// 1st create default pool that returns memory back to callback,
// then use keepMemTillDestroy policy
for (int keep=0; keep<2; keep++) {
getMemCalls = putMemCalls = 0;
if (keep)
pol.keepAllMemory = 1;
pool_create_v1(0, &pol, &pool);
for (int i=0; i<2*ITERS; i+=2) {
ptrs[i] = pool_malloc(pool, 7*1024);
ptrs[i+1] = pool_malloc(pool, 10*1024);
}
ptrs[2*ITERS] = pool_malloc(pool, 8*1024*1024);
ASSERT(!putMemCalls, NULL);
for (int i=0; i<2*ITERS; i++)
pool_free(pool, ptrs[i]);
pool_free(pool, ptrs[2*ITERS]);
size_t totalPutMemCalls = putMemCalls;
if (keep)
ASSERT(!putMemCalls, NULL);
else {
ASSERT(putMemCalls, NULL);
putMemCalls = 0;
}
size_t currGetCalls = getMemCalls;
pool_malloc(pool, 8*1024*1024);
if (keep)
ASSERT(currGetCalls == getMemCalls, "Must not lead to new getMem call");
size_t currPuts = putMemCalls;
pool_reset(pool);
ASSERT(currPuts == putMemCalls, "Pool is not releasing memory during reset.");
pool_destroy(pool);
ASSERT(putMemCalls, NULL);
totalPutMemCalls += putMemCalls;
ASSERT(getMemCalls == totalPutMemCalls, "Memory leak detected.");
}
}
void TestBackend()
{
rml::MemPoolPolicy pol(getMallocMem, putMallocMem);
rml::MemoryPool *mPool;
pool_create_v1(0, &pol, &mPool);
rml::internal::ExtMemoryPool *ePool =
&((rml::internal::MemoryPool*)mPool)->extMemPool;
rml::internal::Backend *backend = &ePool->backend;
for( int p=MaxThread; p>=MinThread; --p ) {
TestBackendWork::initBarrier(p);
NativeParallelFor( p, TestBackendWork(backend) );
}
BlockI *block = backend->getSlabBlock(1);
ASSERT(block, "Memory was not allocated");
backend->putSlabBlock(block);
pool_destroy(mPool);
}
void TestPoolReset()
{
rml::MemPoolPolicy pol(getMallocMem, putMallocMem);
rml::MemoryPool *pool;
pool_create_v1(0, &pol, &pool);
for (int i=0; i<100; i++) {
ASSERT(pool_malloc(pool, 8), NULL);
ASSERT(pool_malloc(pool, 50*1024), NULL);
}
int regionsBeforeReset = liveRegions;
pool_reset(pool);
for (int i=0; i<100; i++) {
ASSERT(pool_malloc(pool, 8), NULL);
ASSERT(pool_malloc(pool, 50*1024), NULL);
}
ASSERT(regionsBeforeReset == liveRegions,
"Expected no new regions allocation.");
pool_destroy(pool);
ASSERT(!liveRegions, "Expected all regions were released.");
}
void TestBackend()
{
rml::MemPoolPolicy pol(getMallocMem, putMallocMem);
rml::MemoryPool *mPool;
pool_create_v1(0, &pol, &mPool);
rml::internal::ExtMemoryPool *ePool =
&((rml::internal::MemoryPool*)mPool)->extMemPool;
rml::internal::Backend *backend = &ePool->backend;
for( int p=MaxThread; p>=MinThread; --p ) {
// regression test against an race condition in backend synchronization,
// triggered only when WhiteboxTestingYield() call yields
for (int i=0; i<100; i++) {
TestBackendWork::initBarrier(p);
NativeParallelFor( p, TestBackendWork(backend) );
}
}
BlockI *block = backend->getSlabBlock(1);
ASSERT(block, "Memory was not allocated");
backend->putSlabBlock(block);
pool_destroy(mPool);
}
void TestPools() {
rml::MemPoolPolicy pol(getMem, putMem);
size_t beforeNumBackRef, afterNumBackRef;
rml::MemoryPool *pool1;
rml::MemoryPool *pool2;
pool_create_v1(0, &pol, &pool1);
pool_create_v1(0, &pol, &pool2);
pool_destroy(pool1);
pool_destroy(pool2);
scalable_allocation_command(TBBMALLOC_CLEAN_ALL_BUFFERS, NULL);
beforeNumBackRef = allocatedBackRefCount();
rml::MemoryPool *fixedPool;
pool_create_v1(0, &pol, &fixedPool);
pol.pAlloc = getMallocMem;
pol.pFree = putMallocMem;
pol.granularity = 8;
rml::MemoryPool *mallocPool;
pool_create_v1(0, &pol, &mallocPool);
/* check that large object cache (LOC) returns correct size for cached objects
passBackendSz Byte objects are cached in LOC, but bypassed the backend, so
memory requested directly from allocation callback.
nextPassBackendSz Byte objects must fit to another LOC bin,
so that their allocation/realeasing leads to cache cleanup.
All this is expecting to lead to releasing of passBackendSz Byte object
from LOC during LOC cleanup, and putMallocMem checks that returned size
is correct.
*/
const size_t passBackendSz = Backend::maxBinned_HugePage+1,
anotherLOCBinSz = minLargeObjectSize+1;
for (int i=0; i<10; i++) { // run long enough to be cached
void *p = pool_malloc(mallocPool, passBackendSz);
ASSERT(p, "Memory was not allocated");
pool_free(mallocPool, p);
}
// run long enough to passBackendSz allocation was cleaned from cache
// and returned back to putMallocMem for size checking
for (int i=0; i<1000; i++) {
void *p = pool_malloc(mallocPool, anotherLOCBinSz);
ASSERT(p, "Memory was not allocated");
pool_free(mallocPool, p);
}
void *smallObj = pool_malloc(fixedPool, 10);
ASSERT(smallObj, "Memory was not allocated");
memset(smallObj, 1, 10);
void *ptr = pool_malloc(fixedPool, 1024);
ASSERT(ptr, "Memory was not allocated");
memset(ptr, 1, 1024);
void *largeObj = pool_malloc(fixedPool, minLargeObjectSize);
ASSERT(largeObj, "Memory was not allocated");
memset(largeObj, 1, minLargeObjectSize);
ptr = pool_malloc(fixedPool, minLargeObjectSize);
ASSERT(ptr, "Memory was not allocated");
memset(ptr, minLargeObjectSize, minLargeObjectSize);
pool_malloc(fixedPool, 10*minLargeObjectSize); // no leak for unsuccesful allocations
pool_free(fixedPool, smallObj);
pool_free(fixedPool, largeObj);
// provoke large object cache cleanup and hope no leaks occurs
for( int p=MaxThread; p>=MinThread; --p )
NativeParallelFor( p, StressLOCacheWork(mallocPool) );
pool_destroy(mallocPool);
pool_destroy(fixedPool);
scalable_allocation_command(TBBMALLOC_CLEAN_ALL_BUFFERS, NULL);
afterNumBackRef = allocatedBackRefCount();
ASSERT(beforeNumBackRef==afterNumBackRef, "backreference leak detected");
{
// test usedSize/cachedSize and LOC bitmask correctness
void *p[5];
pool_create_v1(0, &pol, &mallocPool);
const LargeObjectCache *loc = &((rml::internal::MemoryPool*)mallocPool)->extMemPool.loc;
p[3] = pool_malloc(mallocPool, minLargeObjectSize+2*LargeObjectCache::largeBlockCacheStep);
for (int i=0; i<10; i++) {
p[0] = pool_malloc(mallocPool, minLargeObjectSize);
p[1] = pool_malloc(mallocPool, minLargeObjectSize+LargeObjectCache::largeBlockCacheStep);
pool_free(mallocPool, p[0]);
pool_free(mallocPool, p[1]);
}
ASSERT(loc->getUsedSize(), NULL);
pool_free(mallocPool, p[3]);
ASSERT(loc->getLOCSize() < 3*(minLargeObjectSize+LargeObjectCache::largeBlockCacheStep), NULL);
const size_t maxLocalLOCSize = LocalLOCImpl<3,30>::getMaxSize();
ASSERT(loc->getUsedSize() <= maxLocalLOCSize, NULL);
for (int i=0; i<3; i++)
p[i] = pool_malloc(mallocPool, minLargeObjectSize+i*LargeObjectCache::largeBlockCacheStep);
size_t currUser = loc->getUsedSize();
ASSERT(!loc->getLOCSize() && currUser >= 3*(minLargeObjectSize+LargeObjectCache::largeBlockCacheStep), NULL);
p[4] = pool_malloc(mallocPool, minLargeObjectSize+3*LargeObjectCache::largeBlockCacheStep);
ASSERT(loc->getUsedSize() - currUser >= minLargeObjectSize+3*LargeObjectCache::largeBlockCacheStep, NULL);
pool_free(mallocPool, p[4]);
ASSERT(loc->getUsedSize() <= currUser+maxLocalLOCSize, NULL);
pool_reset(mallocPool);
ASSERT(!loc->getLOCSize() && !loc->getUsedSize(), NULL);
pool_destroy(mallocPool);
}
// To test LOC we need bigger lists than released by current LocalLOC
// in production code. Create special LocalLOC.
{
LocalLOCImpl<2, 20> lLOC;
pool_create_v1(0, &pol, &mallocPool);
rml::internal::ExtMemoryPool *mPool = &((rml::internal::MemoryPool*)mallocPool)->extMemPool;
const LargeObjectCache *loc = &((rml::internal::MemoryPool*)mallocPool)->extMemPool.loc;
for (int i=0; i<22; i++) {
void *o = pool_malloc(mallocPool, minLargeObjectSize+i*LargeObjectCache::largeBlockCacheStep);
bool ret = lLOC.put(((LargeObjectHdr*)o - 1)->memoryBlock, mPool);
ASSERT(ret, NULL);
o = pool_malloc(mallocPool, minLargeObjectSize+i*LargeObjectCache::largeBlockCacheStep);
ret = lLOC.put(((LargeObjectHdr*)o - 1)->memoryBlock, mPool);
ASSERT(ret, NULL);
}
lLOC.externalCleanup(mPool);
ASSERT(!loc->getUsedSize(), NULL);
pool_destroy(mallocPool);
}
}
