void operator()(int id) const {
if (!id) {
backrefGrowthDone = false;
barrier.wait();
for (int i=0; i<BACKREF_GROWTH_ITERS; i++)
ptrs[i] = scalable_malloc(minLargeObjectSize);
backrefGrowthDone = true;
for (int i=0; i<BACKREF_GROWTH_ITERS; i++)
scalable_free(ptrs[i]);
} else {
void *p2 = scalable_malloc(minLargeObjectSize-1);
char *p1 = (char*)scalable_malloc(minLargeObjectSize-1);
LargeObjectHdr *hdr =
(LargeObjectHdr*)(p1+minLargeObjectSize-1 - sizeof(LargeObjectHdr));
hdr->backRefIdx.master = 7;
hdr->backRefIdx.largeObj = 1;
hdr->backRefIdx.offset = 2000;
barrier.wait();
while (!backrefGrowthDone) {
scalable_free(p2);
p2 = scalable_malloc(minLargeObjectSize-1);
}
scalable_free(p1);
scalable_free(p2);
}
}
extern "C" void threadDtor(void*) {
// First, release memory that was allocated before;
// it will not re-initialize the thread-local data if already deleted
prevSmall = currSmall;
scalable_free(currSmall);
prevLarge = currLarge;
scalable_free(currLarge);
// Then, allocate more memory.
// It will re-initialize the allocator data in the thread.
scalable_free(scalable_malloc(8));
}
void operator()( int /*mynum*/ ) const {
CheckNotCached checkNotCached(memSize);
for (size_t n = minLargeObjectSize; n < 5*1024*1024; n += 128*1024)
scalable_free(scalable_malloc(n));
barrier.wait(checkNotCached);
}
void operator delete (void* ptr, const std::nothrow_t&)
{
if (ptr != NULL)
{
scalable_free(ptr);
}
}
void operator delete (void* ptr)
{
if (ptr != NULL)
{
scalable_free(ptr);
}
}
void operator() (int) const {
tbb::internal::spin_wait_while_eq(gPtr, (void*)NULL);
scalable_free(gPtr);
my_barr->wait();
my_ward.wait_to_finish();
++FinishedTasks;
}
void TestHeapLimit()
{
if(!isMallocInitialized()) doInitialization();
// tiny limit to stop caching
int res = scalable_allocation_mode(TBBMALLOC_SET_SOFT_HEAP_LIMIT, 1);
ASSERT(res == TBBMALLOC_OK, NULL);
// provoke bootstrap heap initialization before recording memory size
scalable_free(scalable_malloc(8));
size_t n, sizeBefore = getMemSize();
// Try to provoke call to OS for memory to check that
// requests are not fulfilled from caches.
// Single call is not enough here because of backend fragmentation.
for (n = minLargeObjectSize; n < 10*1024*1024; n += 16*1024) {
void *p = scalable_malloc(n);
bool leave = (sizeBefore != getMemSize());
scalable_free(p);
if (leave)
break;
ASSERT(sizeBefore == getMemSize(), "No caching expected");
}
ASSERT(n < 10*1024*1024, "scalable_malloc doesn't provoke OS request for memory, "
"is some internal cache still used?");
// estimate number of objects in single bootstrap block
int objInBootstrapHeapBlock = (slabSize-2*estimatedCacheLineSize)/sizeof(TLSData);
// When we have more threads than objects in bootstrap heap block,
// additional block can be allocated from a region that is different
// from the original region. Thus even after all caches cleaned,
// we unable to reach sizeBefore.
ASSERT_WARNING(MaxThread<=objInBootstrapHeapBlock,
"The test might fail for larger thread number, "
"as bootstrap heap is not released till size checking.");
for( int p=MaxThread; p>=MinThread; --p ) {
RunTestHeapLimit::initBarrier( p );
NativeParallelFor( p, RunTestHeapLimit(sizeBefore) );
}
// it's try to match limit as well as set limit, so call here
res = scalable_allocation_mode(TBBMALLOC_SET_SOFT_HEAP_LIMIT, 1);
ASSERT(res == TBBMALLOC_OK, NULL);
size_t m = getMemSize();
ASSERT(sizeBefore == m, NULL);
// restore default
res = scalable_allocation_mode(TBBMALLOC_SET_SOFT_HEAP_LIMIT, 0);
ASSERT(res == TBBMALLOC_OK, NULL);
}
void operator()(int) const {
void *objsSmall[ITERS], *objsLarge[ITERS];
for (int i=0; i<ITERS; i++) {
objsSmall[i] = scalable_malloc(minLargeObjectSize-1);
objsLarge[i] = scalable_malloc(minLargeObjectSize);
}
for (int i=0; i<ITERS; i++) {
scalable_free(objsSmall[i]);
scalable_free(objsLarge[i]);
}
#ifdef USE_WINTHREAD
// Under Windows DllMain is used for mallocThreadShutdownNotification
// calling. As DllMain is not used during whitebox testing,
// we have to call the callback manually.
__TBB_mallocThreadShutdownNotification();
#endif
}
void FMallocTBB::Free( void* Ptr )
{
if( !Ptr )
{
return;
}
MEM_TIME(MemTime -= FPlatformTime::Seconds())
#if UE_BUILD_DEBUG || UE_BUILD_DEVELOPMENT
FMemory::Memset(Ptr, DEBUG_FILL_FREED, scalable_msize(Ptr));
#endif
IncrementTotalFreeCalls();
scalable_free(Ptr);
MEM_TIME(MemTime += FPlatformTime::Seconds())
}
extern "C" void callDll()
{
static const int NUM = 20;
void *ptrs[NUM];
for (int i=0; i<NUM; i++) {
ptrs[i] = scalable_malloc(i*1024);
ASSERT(ptrs[i], NULL);
}
for (int i=0; i<NUM; i++)
scalable_free(ptrs[i]);
#if __TBB_SOURCE_DIRECTLY_INCLUDED && (_WIN32||_WIN64)
__TBB_mallocThreadShutdownNotification();
#endif
}
// regression test against incorrect work of msize/realloc
// for aligned objects
void TestAlignedMsize()
{
const int NUM = 4;
char *p[NUM];
size_t objSizes[NUM];
size_t allocSz[] = {4, 8, 512, 2*1024, 4*1024, 8*1024, 16*1024, 0};
size_t align[] = {8, 512, 2*1024, 4*1024, 8*1024, 16*1024, 0};
for (int a=0; align[a]; a++)
for (int s=0; allocSz[s]; s++) {
for (int i=0; i<NUM; i++) {
p[i] = (char*)scalable_aligned_malloc(allocSz[s], align[a]);
ASSERT(is_aligned(p[i], align[a]), NULL);
}
for (int i=0; i<NUM; i++) {
objSizes[i] = scalable_msize(p[i]);
ASSERT(objSizes[i] >= allocSz[s],
"allocated size must be not less than requested");
memset(p[i], i, objSizes[i]);
}
for (int i=0; i<NUM; i++) {
for (unsigned j=0; j<objSizes[i]; j++)
ASSERT(((char*)p[i])[j] == i, "Error: data broken");
}
for (int i=0; i<NUM; i++) {
p[i] = (char*)scalable_aligned_realloc(p[i], 2*allocSz[s], align[a]);
ASSERT(is_aligned(p[i], align[a]), NULL);
memset((char*)p[i]+allocSz[s], i+1, allocSz[s]);
}
for (int i=0; i<NUM; i++) {
for (unsigned j=0; j<allocSz[s]; j++)
ASSERT(((char*)p[i])[j] == i, "Error: data broken");
for (size_t j=allocSz[s]; j<2*allocSz[s]; j++)
ASSERT(((char*)p[i])[j] == i+1, "Error: data broken");
}
for (int i=0; i<NUM; i++)
scalable_free(p[i]);
}
}
bool TestReallocMsize(size_t startSz) {
bool passed = true;
char *buf = (char*)scalable_malloc(startSz);
ASSERT(buf, "");
size_t realSz = scalable_msize(buf);
ASSERT(realSz>=startSz, "scalable_msize must be not less then allocated size");
memset(buf, 'a', realSz-1);
buf[realSz-1] = 0;
char *buf1 = (char*)scalable_realloc(buf, 2*realSz);
ASSERT(buf1, "");
ASSERT(scalable_msize(buf1)>=2*realSz,
"scalable_msize must be not less then allocated size");
buf1[2*realSz-1] = 0;
if ( strspn(buf1, "a") < realSz-1 ) {
REPORT( "Error: data broken for %d Bytes object.\n", startSz);
passed = false;
}
scalable_free(buf1);
return passed;
}
int main(void) {
size_t i, j;
void *p1, *p2;
for( i=0; i<=1<<16; ++i) {
p1 = scalable_malloc(i);
if( !p1 )
printf("Warning: there should be memory but scalable_malloc returned NULL\n");
scalable_free(p1);
}
p1 = p2 = NULL;
for( i=1024*1024; ; i/=2 )
{
scalable_free(p1);
p1 = scalable_realloc(p2, i);
p2 = scalable_calloc(i, 32);
if (p2) {
if (i<sizeof(size_t)) {
for (j=0; j<i; j++)
assert(0==*((char*)p2+j));
} else {
for (j=0; j<i; j+=sizeof(size_t))
assert(0==*((size_t*)p2+j));
}
}
scalable_free(p2);
p2 = scalable_malloc(i);
if (i==0) break;
}
for( i=1; i<1024*1024; i*=2 )
{
scalable_free(p1);
p1 = scalable_realloc(p2, i);
p2 = scalable_malloc(i);
}
scalable_free(p1);
scalable_free(p2);
printf("done\n");
return 0;
}
void ser_solve_tsp_rec(const int (*A)[SIZE], list_node* partial_solution, int position, int partial_weight)
{
#ifdef PARTIAL_WEIGHT_CUTOFF
#ifdef NO_LOCKS
if (partial_weight >= my_weight.local() ) return;
#else
if (partial_weight >= solution_weight) return;
#endif
#endif
int i;
for(i=0; i<SIZE; i++){
// 1. check if i was already used in the solution so far.
// if so, skip it.
int used = 0;
int j = position;
list_node* tmp_p = partial_solution;
while(tmp_p != NULL) {
if (tmp_p->val == i) {
used = 1;
break;
}
tmp_p = tmp_p->next;
}
if (!used) {
//partial_solution[position] = i;
// allocate a node
tmp_p = (list_node*) scalable_malloc(sizeof(list_node));
#ifdef DEBUG
if(tmp_p==NULL) {out_of_memory++;//atomic}
#endif
if (tmp_p != NULL) {
int myweight;
tmp_p->next = partial_solution;
tmp_p->val = i;
myweight = partial_weight + A[partial_solution->val][i];
if (position==SIZE-1) {
// 2a. termination
#ifdef NO_LOCKS
myweight += A[i][0];
if (myweight < my_weight.local() ) {
my_weight.local() = myweight;
}
#else
myweight += A[i][0];
#ifdef DOUBLE_CHECK
if (myweight < solution_weight) {
#endif
// acquire lock
tbb::mutex::scoped_lock myLock(solution_lock);
if (myweight < solution_weight) {
// update solution
solution_weight = myweight;
solution = tmp_p;
}
// implicit lock release
#ifdef DOUBLE_CHECK
}
#endif
#endif
} else {
// 2b. recursion
ser_solve_tsp_rec(A, tmp_p, position+1, myweight);
}
scalable_free (tmp_p);
}
} // end if(!used)
} // end spawn
}
void solve_tsp_rec(const int (*A)[SIZE], list_node* partial_solution, int position, int partial_weight);
class tspBody {
public:
//int* A[SIZE];
const int (*A)[SIZE];
list_node* partial_solution;
int position;
int partial_weight;
void operator () (const tbb::blocked_range<int> &range) const {
int i;
for (i=range.begin(); i<range.end(); i++) {
// 1. check if node i was already used in the solution so far.
// if so, skip it.
int used = 0;
int j = position;
list_node* tmp_p = partial_solution;
while(tmp_p != NULL) {
if (tmp_p->val == i) {
used = 1;
break;
}
tmp_p = tmp_p->next;
}
if (!used) {
//partial_solution[position] = i;
// allocate a node
tmp_p = (list_node*) scalable_malloc( sizeof(list_node) );
#ifdef DEBUG
if(tmp_p==NULL) {out_of_memory++;//atomic}
#endif
if (tmp_p != NULL) {
int myweight;
tmp_p->next = partial_solution;
tmp_p->val = i;
myweight = partial_weight + A[partial_solution->val][i];
if (position==SIZE-1) {
// 2a. termination
#ifdef NO_LOCKS
myweight += A[i][0];
if (myweight < my_weight.local() ) {
my_weight.local() = myweight;
}
#else
myweight += A[i][0];
#ifdef DOUBLE_CHECK
if (myweight < solution_weight) {
#endif
tbb::mutex::scoped_lock myLock(solution_lock);
if (myweight < solution_weight) {
// lock solution
solution_weight = myweight;
solution = tmp_p;
}
// implicit lock release
#ifdef DOUBLE_CHECK
}
#endif
#endif
} else {
// 2b. recursion
#ifdef CUTOFF
if (position+1>=CUTOFF_LEVEL)
ser_solve_tsp_rec(A, tmp_p, position+1, myweight);
else
solve_tsp_rec(A, tmp_p, position+1, myweight);
#else
solve_tsp_rec(A, tmp_p, position+1, myweight);
#endif
}
scalable_free (tmp_p);
}
} // end if(!used)
} // end for loop
}
// Constructor
tspBody(const int A[SIZE][SIZE], list_node* partial_solution, int position, int partial_weight):
A(A), partial_solution(partial_solution), position(position), partial_weight(partial_weight) {}
};
int main(void) {
size_t i, j;
int curr_mode, res;
void *p1, *p2;
atexit( MyExit );
for ( curr_mode = 0; curr_mode<=1; curr_mode++) {
assert(ExpectedResultHugePages ==
scalable_allocation_mode(TBBMALLOC_USE_HUGE_PAGES, !curr_mode));
p1 = scalable_malloc(10*1024*1024);
assert(p1);
assert(ExpectedResultHugePages ==
scalable_allocation_mode(TBBMALLOC_USE_HUGE_PAGES, curr_mode));
scalable_free(p1);
}
/* note that huge pages (if supported) are still enabled at this point */
#if __TBB_SOURCE_DIRECTLY_INCLUDED
assert(TBBMALLOC_OK ==
scalable_allocation_mode(TBBMALLOC_INTERNAL_SOURCE_INCLUDED, 0));
#endif
for( i=0; i<=1<<16; ++i) {
p1 = scalable_malloc(i);
if( !p1 )
printf("Warning: there should be memory but scalable_malloc returned NULL\n");
scalable_free(p1);
}
p1 = p2 = NULL;
for( i=1024*1024; ; i/=2 )
{
scalable_free(p1);
p1 = scalable_realloc(p2, i);
p2 = scalable_calloc(i, 32);
if (p2) {
if (i<sizeof(size_t)) {
for (j=0; j<i; j++)
assert(0==*((char*)p2+j));
} else {
for (j=0; j<i; j+=sizeof(size_t))
assert(0==*((size_t*)p2+j));
}
}
scalable_free(p2);
p2 = scalable_malloc(i);
if (i==0) break;
}
for( i=1; i<1024*1024; i*=2 )
{
scalable_free(p1);
p1 = scalable_realloc(p2, i);
p2 = scalable_malloc(i);
}
scalable_free(p1);
scalable_free(p2);
res = scalable_allocation_command(TBBMALLOC_CLEAN_ALL_BUFFERS, NULL);
assert(res == TBBMALLOC_OK);
res = scalable_allocation_command(TBBMALLOC_CLEAN_THREAD_BUFFERS, NULL);
/* expect all caches cleaned before, so got nothing from CLEAN_THREAD_BUFFERS */
assert(res == TBBMALLOC_NO_EFFECT);
/* check that invalid param argument give expected result*/
res = scalable_allocation_command(TBBMALLOC_CLEAN_THREAD_BUFFERS,
(void*)(intptr_t)1);
assert(res == TBBMALLOC_INVALID_PARAM);
__TBB_mallocProcessShutdownNotification();
printf("done\n");
return 0;
}
/* test that it's possible to call allocation function from atexit
after mallocProcessShutdownNotification() called */
static void MyExit(void) {
void *p = scalable_malloc(32);
assert(p);
scalable_free(p);
__TBB_mallocProcessShutdownNotification();
}
void operator delete[](void* ptr, const std::nothrow_t&) throw()
{
scalable_free(ptr);
}
void operator delete[](void* ptr) throw()
{
scalable_free(ptr);
}
void clear() {
scalable_free(p);
}
void TestObjectRecognition() {
size_t headersSize = sizeof(LargeMemoryBlock)+sizeof(LargeObjectHdr);
unsigned falseObjectSize = 113; // unsigned is the type expected by getObjectSize
size_t obtainedSize;
ASSERT(sizeof(BackRefIdx)==4, "Unexpected size of BackRefIdx");
ASSERT(getObjectSize(falseObjectSize)!=falseObjectSize, "Error in test: bad choice for false object size");
void* mem = scalable_malloc(2*slabSize);
ASSERT(mem, "Memory was not allocated");
Block* falseBlock = (Block*)alignUp((uintptr_t)mem, slabSize);
falseBlock->objectSize = falseObjectSize;
char* falseSO = (char*)falseBlock + falseObjectSize*7;
ASSERT(alignDown(falseSO, slabSize)==(void*)falseBlock, "Error in test: false object offset is too big");
void* bufferLOH = scalable_malloc(2*slabSize + headersSize);
ASSERT(bufferLOH, "Memory was not allocated");
LargeObjectHdr* falseLO =
(LargeObjectHdr*)alignUp((uintptr_t)bufferLOH + headersSize, slabSize);
LargeObjectHdr* headerLO = (LargeObjectHdr*)falseLO-1;
headerLO->memoryBlock = (LargeMemoryBlock*)bufferLOH;
headerLO->memoryBlock->unalignedSize = 2*slabSize + headersSize;
headerLO->memoryBlock->objectSize = slabSize + headersSize;
headerLO->backRefIdx = BackRefIdx::newBackRef(/*largeObj=*/true);
setBackRef(headerLO->backRefIdx, headerLO);
ASSERT(scalable_msize(falseLO) == slabSize + headersSize,
"Error in test: LOH falsification failed");
removeBackRef(headerLO->backRefIdx);
const int NUM_OF_IDX = BR_MAX_CNT+2;
BackRefIdx idxs[NUM_OF_IDX];
for (int cnt=0; cnt<2; cnt++) {
for (int master = -10; master<10; master++) {
falseBlock->backRefIdx.master = (uint16_t)master;
headerLO->backRefIdx.master = (uint16_t)master;
for (int bl = -10; bl<BR_MAX_CNT+10; bl++) {
falseBlock->backRefIdx.offset = (uint16_t)bl;
headerLO->backRefIdx.offset = (uint16_t)bl;
for (int largeObj = 0; largeObj<2; largeObj++) {
falseBlock->backRefIdx.largeObj = largeObj;
headerLO->backRefIdx.largeObj = largeObj;
obtainedSize = safer_scalable_msize(falseSO, NULL);
ASSERT(obtainedSize==0, "Incorrect pointer accepted");
obtainedSize = safer_scalable_msize(falseLO, NULL);
ASSERT(obtainedSize==0, "Incorrect pointer accepted");
}
}
}
if (cnt == 1) {
for (int i=0; i<NUM_OF_IDX; i++)
removeBackRef(idxs[i]);
break;
}
for (int i=0; i<NUM_OF_IDX; i++) {
idxs[i] = BackRefIdx::newBackRef(/*largeObj=*/false);
setBackRef(idxs[i], NULL);
}
}
char *smallPtr = (char*)scalable_malloc(falseObjectSize);
obtainedSize = safer_scalable_msize(smallPtr, NULL);
ASSERT(obtainedSize==getObjectSize(falseObjectSize), "Correct pointer not accepted?");
scalable_free(smallPtr);
obtainedSize = safer_scalable_msize(mem, NULL);
ASSERT(obtainedSize>=2*slabSize, "Correct pointer not accepted?");
scalable_free(mem);
scalable_free(bufferLOH);
}
void operator delete ( void * ptr ) throw()
{
if( ptr != 0 ) scalable_free ( ptr ) ;
}
void operator delete ( void * ptr , const std::nothrow_t & ) throw()
{
if( ptr != 0 ) scalable_free ( ptr ) ;
}
void operator() ( int ) const {
barrier.wait();
for( int i = deallocs_counter++; i < num_allocs; i = deallocs_counter++ )
scalable_free( ptrs[i] );
}