int TestMain () {
int i;
ptrdiff_t memory_leak;
// warm-up run
NativeParallelFor( 1, Run() );
/* 1st call to GetMemoryUsage() allocate some memory,
but it seems memory consumption stabilized after this.
*/
GetMemoryUsage();
size_t memory_in_use = GetMemoryUsage();
ASSERT(memory_in_use == GetMemoryUsage(),
"Memory consumption should not increase after 1st GetMemoryUsage() call");
// expect that memory consumption stabilized after several runs
for (i=0; i<3; i++) {
size_t memory_in_use = GetMemoryUsage();
for (int j=0; j<10; j++)
NativeParallelFor( 1, Run() );
memory_leak = GetMemoryUsage() - memory_in_use;
if (memory_leak == 0) // possibly too strong?
break;
}
if(3==i) {
// not stabilized, could be leak
REPORT( "Error: memory leak of up to %ld bytes\n", static_cast<long>(memory_leak));
exit(1);
}
return Harness::Done;
}
int TestMain () {
const int ITERS = 20;
int i;
std::ptrdiff_t memory_leak = 0;
GetMemoryUsage();
for (int run = 0; run<2; run++) {
// expect that memory consumption stabilized after several runs
for (i=0; i<ITERS; i++) {
std::size_t memory_in_use = GetMemoryUsage();
if (run)
LoadThreadsUnload();
else
ThreadsLoadUnload();
memory_leak = GetMemoryUsage() - memory_in_use;
if (memory_leak == 0) // possibly too strong?
break;
}
if(i==ITERS) {
// not stabilized, could be leak
REPORT( "Error: memory leak of up to %ld bytes\n", static_cast<long>(memory_leak));
exit(1);
}
}
return Harness::Done;
}
void
CSeqDBPerfApp::x_UpdateMemoryUsage(const int thread_id /* = 0 */)
{
SMemUsage mu;
if (GetMemoryUsage(&mu.total, &mu.resident, &mu.shared)) {
m_MemoryUsage[thread_id] += mu;
}
}
bool TestBootstrapLeak() {
/* In the bug 1518, each thread leaked ~384 bytes.
Initially, scalable allocator maps 1MB. Thus it is necessary to take out most of this space.
1MB is chunked into 16K blocks; of those, one block is for thread bootstrap, and one more
should be reserved for the test body. 62 blocks left, each can serve 15 objects of 1024 bytes.
*/
const int alloc_size = 1024;
const int take_out_count = 15*62;
tbb::scalable_allocator<char> a;
char* array[take_out_count];
for( int i=0; i<take_out_count; ++i )
array[i] = a.allocate( alloc_size );
RunThread( minimalAllocFree(), alloc_size ); // for threading library to take some memory
size_t memory_in_use = GetMemoryUsage();
// Wait for memory usage data to "stabilize". The test number (1000) has nothing underneath.
for( int i=0; i<1000; i++) {
if( GetMemoryUsage()!=memory_in_use ) {
memory_in_use = GetMemoryUsage();
i = -1;
}
}
ptrdiff_t memory_leak = 0;
// Note that 16K bootstrap memory block is enough to serve 42 threads.
const int num_thread_runs = 200;
for (int run=0; run<3; run++) {
memory_in_use = GetMemoryUsage();
for( int i=0; i<num_thread_runs; ++i )
RunThread( minimalAllocFree(), alloc_size );
memory_leak = GetMemoryUsage() - memory_in_use;
if (!memory_leak)
break;
}
if( memory_leak>0 ) { // possibly too strong?
REPORT( "Error: memory leak of up to %ld bytes\n", static_cast<long>(memory_leak));
}
for( int i=0; i<take_out_count; ++i )
a.deallocate( array[i], alloc_size );
return memory_leak<=0;
}
void CheckReallocLeak()
{
int i;
const int ITER_TO_STABILITY = 10;
// do bootstrap
for (int k=0; k<3; k++)
InvariantDataRealloc(/*aligned=*/false, 128*MByte, /*checkData=*/false);
size_t prev = GetMemoryUsage(peakUsage);
// expect realloc to not increase peak memory consumption after ITER_TO_STABILITY-1 iterations
for (i=0; i<ITER_TO_STABILITY; i++) {
for (int k=0; k<3; k++)
InvariantDataRealloc(/*aligned=*/false, 128*MByte, /*checkData=*/false);
size_t curr = GetMemoryUsage(peakUsage);
if (prev == curr)
break;
prev = curr;
}
ASSERT(i < ITER_TO_STABILITY, "Can't stabilize memory consumption.");
}
bool CTestTlsObjectApp::Thread_Run(int /*idx*/)
{
try {
RunTest();
RunTest();
RunTest();
return true;
}
NCBI_CATCH_ALL("Test failed");
size_t total, resident, shared;
if ( GetMemoryUsage(&total, &resident, &shared) ) {
ERR_POST("Alloc: "<<alloc_count.Get()<<" "<<object_count.Get()<<'\n'<<
"Memory: "<<total<<" "<<resident<<" "<<shared);
}
return false;
}
void CD3DTexture::SaveTexture()
{
if (m_texture)
{
delete[] m_data;
m_data = nullptr;
ID3D11DeviceContext* pContext = g_Windowing.GetImmediateContext();
D3D11_TEXTURE2D_DESC textureDesc;
m_texture->GetDesc(&textureDesc);
ID3D11Texture2D* texture = nullptr;
if (textureDesc.Usage != D3D11_USAGE_STAGING || 0 == (textureDesc.CPUAccessFlags & D3D11_CPU_ACCESS_READ))
{
// create texture which can be readed by CPU - D3D11_USAGE_STAGING
CD3D11_TEXTURE2D_DESC stagingDesc(textureDesc);
stagingDesc.Usage = D3D11_USAGE_STAGING;
stagingDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
stagingDesc.BindFlags = 0;
if (FAILED(g_Windowing.Get3D11Device()->CreateTexture2D(&stagingDesc, NULL, &texture)))
return;
// copy contents to new texture
pContext->CopyResource(texture, m_texture);
}
else
texture = m_texture;
// read data from texture
D3D11_MAPPED_SUBRESOURCE res;
if (SUCCEEDED(pContext->Map(texture, 0, D3D11_MAP_READ, 0, &res)))
{
m_pitch = res.RowPitch;
unsigned int memUsage = GetMemoryUsage(res.RowPitch);
m_data = new unsigned char[memUsage];
memcpy(m_data, res.pData, memUsage);
pContext->Unmap(texture, 0);
}
else
CLog::Log(LOGERROR, "%s - Failed to store resource.", __FUNCTION__);
if (texture != m_texture)
SAFE_RELEASE(texture);
}
}
void PrintMemoryUsage(const std::string& description, const std::string& filename) {
#ifdef HAVE_MUELU_PROC_SELF_STATUS
std::cout << description << ": " << GetMemoryUsage() << std::endl;
#endif
#ifdef HAVE_MUELU_GOOGLE_PERFTOOLS
if (IsHeapProfilerRunning()) {
char* profile = GetHeapProfile();
std::istringstream iss(profile);
std::string sub;
iss >> sub;
iss >> sub;
iss >> sub; // skip 3 first substring
iss >> sub;
double MB = atof(sub.c_str()) / (1024*1024);
// print
if (description != "") {
std::ostringstream sname;
sname.precision(1);
sname << description << ": " << std::fixed << MB << " MB";
std::cout << sname.str() << std::endl;
}
// dump to file
if (filename != "") {
std::ofstream out(filename.c_str(), std::ios::out | std::ios::binary);
if(!out) {
std::cout << "Cannot open output file: " << filename << std::endl;
return;
}
out.write(profile, strlen(profile));
out.close();
// dump to file using HeapProfilerDump:
// HeapProfilerDump(filename.c_str());
}
free(profile);
}
#endif
}
int TestMain () {
bool passed = true;
// Check whether memory usage data can be obtained; if not, skip test_bootstrap_leak.
if( GetMemoryUsage() )
passed &= TestBootstrapLeak();
for (size_t a=1, b=1, sum=1; sum<=64*1024; ) {
passed &= TestReallocMsize(sum);
a = b;
b = sum;
sum = a+b;
}
for (size_t a=2; a<=64*1024; a*=2)
passed &= TestReallocMsize(a);
ASSERT( passed, "Test failed" );
return Harness::Done;
}
// ============================================================================
void
DumpMemory(
const string& prefix)
// ============================================================================
{
Uint8 totalMemory = GetPhysicalMemorySize();
size_t usedMemory; size_t residentMemory; size_t sharedMemory;
if (!GetMemoryUsage(&usedMemory, &residentMemory, &sharedMemory)) {
cerr << "Unable to get memory counts!" << endl;
}
else {
cerr << prefix
<< "Total:" << totalMemory
<< " Used:" << usedMemory << "("
<< (100*usedMemory)/totalMemory <<"%)"
<< " Resident:" << residentMemory << "("
<< int((100.0*residentMemory)/totalMemory) <<"%)"
<< endl;
}
}
void message(const char* msg,
const char* msg1, double t1,
const char* msg2, double t2,
size_t COUNT)
{
if ( 0 ) {
LOG_POST(msg
<<'\n'<<setw(40) << msg1 << ": "<< t1 * 1e9/(double)COUNT << " usec"
<<'\n'<<setw(40) << msg2 << ": "<< t2 * 1e9/(double)COUNT << " usec");
}
size_t total, resident, shared;
if ( GetMemoryUsage(&total, &resident, &shared) ) {
max_total = max(max_total, total);
max_resident = max(max_resident, resident);
max_shared = max(max_shared, shared);
if ( 0 ) {
LOG_POST("Alloc: "<<alloc_count.Get()<<
" "<<object_count.Get()<<'\n'<<
"Current memory: "<<total<<" "<<resident<<" "<<shared);
}
}
}
int main(int argc, char** argv)
{
if(!opt.GenInputMgr(argc,argv))
return false;
CGspSystem::InitSystem();
if(opt._c == 0 || opt._c == 1)
{
omp_set_num_threads(opt._t);
PayloadFileCollection *pyCol;
if(opt._c == 0)
{
pyCol = new PayloadFileCollection();
}
else
{
pyCol = new PayloadPackFileCollection();
}
if(!pyCol->LoadCollectionFromPath(opt._s.c_str(),opt._k.size()>0?opt._k.c_str():0))
{
opt.GenShowUsage();
printf("Loading file path %s failed!",opt._s.c_str());
if(opt._k.size()>0)
printf("Key stirng is %s",opt._k.c_str());
printf("\n");
return 0;
}
ElementSequenceCol seqCol;
seqCol.setParam(opt._i,opt._x,opt._r == -1?RETAIN_LENGTH:opt._r);
AuditSequence adMap;
adMap.setParam(opt._w,opt._n,opt._p);
FragAllLevCol fragCol;
fragCol.setParam(opt._f,opt._o,opt._l,opt._z);
CGspMining mine;
if(!mine.Init(opt._a.c_str(), pyCol, &seqCol,&adMap,&fragCol))
{
printf("Mining init failed!\n");
}
else
{
mine.execute();
mine.Destroy();
}
delete pyCol;
}
else if(opt._c == 2)
{
printf("Begin packing file...\n");
printf("\tPacking information:\n");
printf("\tSource index file path:%s\n",opt._s.c_str());
printf("\tDestination folder path:%s\n",opt._d.c_str());
printf("\tRetain parameter:%d\n",opt._r);
CPack pack;
if(!pack.InitPack(opt._s.c_str(),opt._d.c_str(),opt._r))
{
printf("To packing a set, init index file %s and dest path %s are failed\n",
opt._s.c_str(),opt._d.c_str());
}
else
{
pack.Execute();
printf("Packing end.\n");
}
}
else if(opt._c == 3)
{
CEvaluate eva;
if(!eva.Init(opt._s.c_str(), opt._e.c_str()))
{
printf("Evaluate failed!\n");
}
eva.Evaluate();
}
CGspSystem::DestroySystem();
printf("Total program costs %f seconds.\n", CClockTime::GetTimeCostFromStartUp());
MemoryState state;
GetMemoryUsage(state);
printf("Peak memory usage: %f MB\n", state._peakWorkingSetSize);
return 0;
}
bool CPdbCache::ReleaseCacheEntry(int nEntry)
{
CAutoLock lock(&m_AccessLock);
// Find an entry with such ID
std::map<int, _CacheEntry>::iterator e_it = m_aEntries.find(nEntry);
if(e_it==m_aEntries.end())
{
// Possible error - invalid argument
return false;
}
if(e_it->second.m_nRefCount==0)
{
return false; // This entry is already released!
}
_CacheEntry& Entry = e_it->second;
Entry.m_nRefCount--; // Decrement ref count
// Check if ref count is zero
if(Entry.m_nRefCount==0)
{
if(Entry.m_bPendingDelete)
{
std::wstring sPath = Entry.m_sPdbPath;
// We can delete the entry right now
if(!EraseCacheEntry(nEntry))
{
// Entry cannot be erased from cache?
return false;
}
#ifdef _WIN32
// We can delete the file right now
return DeletePdbFile(sPath);
#else
// In Linux, we have already deleted the hardlinks to file,
// so just return now.
return true;
#endif
}
size_t uMemUsageKB = GetMemoryUsage();
if(uMemUsageKB==0 || uMemUsageKB>m_uMaxMemUsageKB)
{
// Reduce memory consumtion
if(Entry.m_pPdbReader!=NULL)
Entry.m_pPdbReader->ReduceMemoryConsumption();
}
// Insert this entry into unref index
time_t CurTime;
time(&CurTime);
Entry.m_AccessTime = CurTime; // update access time of the entry
std::pair<time_t, int> Pair(CurTime, nEntry);
m_aUnrefIndex.insert(Pair);
}
// Ensure cache size is below the limit
CheckCacheOverflow();
// OK
return true;
}
void CWNJobWatcher::Notify(const CWorkerNodeJobContext& job_context,
EEvent event)
{
switch (event) {
case eJobStarted:
{
CMutexGuard guard(m_ActiveJobsMutex);
m_ActiveJobs[const_cast<CWorkerNodeJobContext*>(&job_context)] =
SJobActivity();
++m_JobsStarted;
if (m_MaxJobsAllowed > 0 && m_JobsStarted > m_MaxJobsAllowed - 1) {
LOG_POST_X(1, "The maximum number of allowed jobs (" <<
m_MaxJobsAllowed << ") has been reached. "
"Sending the shutdown request." );
CGridGlobals::GetInstance().
RequestShutdown(CNetScheduleAdmin::eNormalShutdown);
}
}
break;
case eJobStopped:
{
CMutexGuard guard(m_ActiveJobsMutex);
m_ActiveJobs.erase(
const_cast<CWorkerNodeJobContext*>(&job_context));
}
break;
case eJobFailed:
++m_JobsFailed;
if (m_MaxFailuresAllowed > 0 &&
m_JobsFailed > m_MaxFailuresAllowed - 1) {
LOG_POST_X(2, "The maximum number of failed jobs (" <<
m_MaxFailuresAllowed << ") has been reached. "
"Shutting down..." );
CGridGlobals::GetInstance().
RequestShutdown(CNetScheduleAdmin::eShutdownImmediate);
}
break;
case eJobSucceeded:
++m_JobsSucceeded;
break;
case eJobReturned:
++m_JobsReturned;
break;
case eJobRescheduled:
++m_JobsRescheduled;
break;
case eJobCanceled:
++m_JobsCanceled;
break;
case eJobLost:
++m_JobsLost;
break;
}
if (event != eJobStarted) {
CGridWorkerNode worker_node(job_context.GetWorkerNode());
Uint8 total_memory_limit = worker_node.GetTotalMemoryLimit();
if (total_memory_limit > 0) { // memory check requested
size_t memory_usage;
if (!GetMemoryUsage(&memory_usage, 0, 0)) {
ERR_POST("Could not check self memory usage" );
} else if (memory_usage > total_memory_limit) {
ERR_POST(Warning << "Memory usage (" << memory_usage <<
") is above the configured limit (" <<
total_memory_limit << ")");
CGridGlobals::GetInstance().RequestShutdown(
CNetScheduleAdmin::eNormalShutdown,
RESOURCE_OVERUSE_EXIT_CODE);
}
}
int total_time_limit = worker_node.GetTotalTimeLimit();
if (total_time_limit > 0 && // time check requested
time(0) > worker_node.GetStartupTime() + total_time_limit)
CGridGlobals::GetInstance().RequestShutdown(
CNetScheduleAdmin::eNormalShutdown, RESOURCE_OVERUSE_EXIT_CODE);
}
}
void cWebAdmin::HandleWebadminRequest(cHTTPConnection & a_Connection, cHTTPRequest & a_Request)
{
if (!a_Request.HasAuth())
{
a_Connection.SendNeedAuth("MCServer WebAdmin");
return;
}
// Check auth:
AString UserPassword = m_IniFile.GetValue("User:"******"Password", "");
if ((UserPassword == "") || (a_Request.GetAuthPassword() != UserPassword))
{
a_Connection.SendNeedAuth("MCServer WebAdmin - bad username or password");
return;
}
// Check if the contents should be wrapped in the template:
AString URL = a_Request.GetBareURL();
ASSERT(URL.length() > 0);
bool ShouldWrapInTemplate = ((URL.length() > 1) && (URL[1] != '~'));
// Retrieve the request data:
cWebadminRequestData * Data = (cWebadminRequestData *)(a_Request.GetUserData());
if (Data == NULL)
{
a_Connection.SendStatusAndReason(500, "Bad UserData");
return;
}
// Wrap it all up for the Lua call:
AString Template;
HTTPTemplateRequest TemplateRequest;
TemplateRequest.Request.Username = a_Request.GetAuthUsername();
TemplateRequest.Request.Method = a_Request.GetMethod();
TemplateRequest.Request.Path = URL.substr(1);
if (Data->m_Form.Finish())
{
for (cHTTPFormParser::const_iterator itr = Data->m_Form.begin(), end = Data->m_Form.end(); itr != end; ++itr)
{
HTTPFormData HTTPfd;
HTTPfd.Value = itr->second;
HTTPfd.Type = "";
HTTPfd.Name = itr->first;
TemplateRequest.Request.FormData[itr->first] = HTTPfd;
TemplateRequest.Request.PostParams[itr->first] = itr->second;
} // for itr - Data->m_Form[]
// Parse the URL into individual params:
size_t idxQM = a_Request.GetURL().find('?');
if (idxQM != AString::npos)
{
cHTTPFormParser URLParams(cHTTPFormParser::fpkURL, a_Request.GetURL().c_str() + idxQM + 1, a_Request.GetURL().length() - idxQM - 1, *Data);
URLParams.Finish();
for (cHTTPFormParser::const_iterator itr = URLParams.begin(), end = URLParams.end(); itr != end; ++itr)
{
TemplateRequest.Request.Params[itr->first] = itr->second;
} // for itr - URLParams[]
}
}
// Try to get the template from the Lua template script
if (ShouldWrapInTemplate)
{
if (m_TemplateScript.Call("ShowPage", this, &TemplateRequest, cLuaState::Return, Template))
{
cHTTPResponse Resp;
Resp.SetContentType("text/html");
a_Connection.Send(Resp);
a_Connection.Send(Template.c_str(), Template.length());
return;
}
a_Connection.SendStatusAndReason(500, "m_TemplateScript failed");
return;
}
AString BaseURL = GetBaseURL(URL);
AString Menu;
Template = "{CONTENT}";
AString FoundPlugin;
for (PluginList::iterator itr = m_Plugins.begin(); itr != m_Plugins.end(); ++itr)
{
cWebPlugin * WebPlugin = *itr;
std::list< std::pair<AString, AString> > NameList = WebPlugin->GetTabNames();
for (std::list< std::pair<AString, AString> >::iterator Names = NameList.begin(); Names != NameList.end(); ++Names)
{
Menu += "<li><a href='" + BaseURL + WebPlugin->GetWebTitle().c_str() + "/" + (*Names).second + "'>" + (*Names).first + "</a></li>";
}
}
sWebAdminPage Page = GetPage(TemplateRequest.Request);
AString Content = Page.Content;
FoundPlugin = Page.PluginName;
if (!Page.TabName.empty())
{
FoundPlugin += " - " + Page.TabName;
}
if (FoundPlugin.empty()) // Default page
{
Content = GetDefaultPage();
}
if (ShouldWrapInTemplate && (URL.size() > 1))
{
Content += "\n<p><a href='" + BaseURL + "'>Go back</a></p>";
}
int MemUsageKiB = GetMemoryUsage();
if (MemUsageKiB > 0)
{
ReplaceString(Template, "{MEM}", Printf("%.02f", (double)MemUsageKiB / 1024));
ReplaceString(Template, "{MEMKIB}", Printf("%d", MemUsageKiB));
}
else
{
ReplaceString(Template, "{MEM}", "unknown");
ReplaceString(Template, "{MEMKIB}", "unknown");
}
ReplaceString(Template, "{USERNAME}", a_Request.GetAuthUsername());
ReplaceString(Template, "{MENU}", Menu);
ReplaceString(Template, "{PLUGIN_NAME}", FoundPlugin);
ReplaceString(Template, "{CONTENT}", Content);
ReplaceString(Template, "{TITLE}", "MCServer");
AString NumChunks;
Printf(NumChunks, "%d", cRoot::Get()->GetTotalChunkCount());
ReplaceString(Template, "{NUMCHUNKS}", NumChunks);
cHTTPResponse Resp;
Resp.SetContentType("text/html");
a_Connection.Send(Resp);
a_Connection.Send(Template.c_str(), Template.length());
}
