/** At startup on each processor, this method is called.
This sets up the converse level comlib strategies.
This is called before any mainchare main functions.
*/
void initConvComlibManager(){
if(!CkpvInitialized(conv_com_object))
CkpvInitialize(ConvComlibManager, conv_com_object);
if(CkpvAccess(conv_com_object).getInitialized()) {
CmiPrintf("Comlib initialized more than once!\n");
return;
}
CkpvInitialize(int, RecvdummyHandle);
CkpvAccess(RecvdummyHandle) = CkRegisterHandler((CmiHandler)recv_dummy);
CkpvInitialize(int, comlib_receive_table);
CkpvAccess(comlib_receive_table) = CkRegisterHandler((CmiHandler)comlibReceiveTableHandler);
CkpvInitialize(int, comlib_table_received);
CkpvAccess(comlib_table_received) = CkRegisterHandler((CmiHandler)comlibTableReceivedHandler);
CkpvInitialize(int, comlib_ready);
CkpvAccess(comlib_ready) = CkRegisterHandler((CmiHandler)comlibReadyHandler);
// init strategy specific variables
// router strategy
CkpvInitialize(int, RouterRecvHandle);
CkpvAccess(RouterRecvHandle) = CkRegisterHandler((CmiHandler)routerRecvManyCombinedMsg);
CkpvInitialize(int, RouterProcHandle);
CkpvAccess(RouterProcHandle) = CkRegisterHandler((CmiHandler)routerProcManyCombinedMsg);
CkpvInitialize(int, RouterDummyHandle);
CkpvAccess(RouterDummyHandle) = CkRegisterHandler((CmiHandler)routerDummyMsg);
// streaming strategy
CpvInitialize(int, streaming_handler_id);
CpvAccess(streaming_handler_id) = CmiRegisterHandler(StreamingHandlerFn);
// mesh streaming strategy
CkpvInitialize(int, streaming_column_handler_id);
CkpvAccess(streaming_column_handler_id) = CkRegisterHandler(streaming_column_handler);
// pipelined broadcast
CkpvInitialize(int, pipeline_handler);
CkpvInitialize(int, pipeline_frag_handler);
CkpvAccess(pipeline_handler) = CkRegisterHandler((CmiHandler)PipelineHandler);
CkpvAccess(pipeline_frag_handler) = CkRegisterHandler((CmiHandler)PipelineFragmentHandler);
// general handler
CkpvInitialize(int, comlib_handler);
CkpvAccess(comlib_handler) = CkRegisterHandler((CmiHandler) strategyHandler);
//PUPable_reg(Strategy); ABSTRACT
//PUPable_reg(ConvComlibInstanceHandle);
if (CmiMyRank() == 0) {
PUPable_reg(RouterStrategy);
PUPable_reg(StreamingStrategy);
PUPable_reg(MeshStreamingStrategy);
PUPable_reg(PipeBroadcastConverse);
PUPable_reg(MessageHolder);
}
CkpvAccess(conv_com_object).setInitialized();
}
/**
For each TraceFoo module, _createTraceFoo() must be defined.
This function is called in _createTraces() generated in moduleInit.C
*/
void _createTraceprojector(char **argv)
{
DEBUGF(("%d createTraceProjector\n", CkMyPe()));
CkpvInitialize(Trace*, _traceproj);
CkpvInitialize(CkVec<UsrEvent *>, usrEvents);
CkpvAccess(_traceproj) = new TraceProjector(argv);
CkpvAccess(_traces)->addTrace(CkpvAccess(_traceproj));
}
Communicate::Communicate(void)
{
CkpvInitialize(CmmTable, CsmMessages);
CsmHandlerIndex = CmiRegisterHandler((CmiHandler) CsmHandler);
CsmAckHandlerIndex = CmiRegisterHandler((CmiHandler) CsmAckHandler);
CkpvAccess(CsmMessages) = CmmNew();
if ( CmiMyNode() * 2 + 2 < CmiNumNodes() ) nchildren = 2;
else if ( CmiMyNode() * 2 + 1 < CmiNumNodes() ) nchildren = 1;
else nchildren = 0;
CkpvInitialize(int, CsmAcks);
CkpvAccess(CsmAcks) = nchildren;
}
void ProcessorPrivateInit(void)
{
CkpvInitialize(AtomMap*, AtomMap_instance);
CkpvAccess(AtomMap_instance) = 0;
CkpvInitialize(BroadcastMgr*, BroadcastMgr_instance);
CkpvAccess(BroadcastMgr_instance) = 0;
CkpvInitialize(CollectionMaster*, CollectionMaster_instance);
CkpvAccess(CollectionMaster_instance) = 0;
CkpvInitialize(CollectionMgr*, CollectionMgr_instance);
CkpvAccess(CollectionMgr_instance) = 0;
CkpvInitialize(LdbCoordinator*, LdbCoordinator_instance);
CkpvAccess(LdbCoordinator_instance) = 0;
CkpvInitialize(Node*, Node_instance);
CkpvAccess(Node_instance) = 0;
CkpvInitialize(PatchMap*, PatchMap_instance);
CkpvAccess(PatchMap_instance) = 0;
CkpvInitialize(PatchMgr*, PatchMgr_instance);
CkpvAccess(PatchMgr_instance) = 0;
CkpvInitialize(ProxyMgr*, ProxyMgr_instance);
CkpvAccess(ProxyMgr_instance) = 0;
CkpvInitialize(ReductionMgr*, ReductionMgr_instance);
CkpvAccess(ReductionMgr_instance) = 0;
CkpvInitialize(PatchMgr*, PatchMap_patchMgr);
CkpvAccess(PatchMap_patchMgr) = 0;
CkpvInitialize(BOCgroup, BOCclass_group);
CkpvInitialize(Communicate*, comm);
CkpvAccess(comm) = 0;
CkpvInitialize(Sync*, Sync_instance);
CkpvAccess(Sync_instance) = 0;
CkpvInitialize(infostream, iout_obj);
#ifdef PROCTRACE_DEBUG
CkpvInitialize(DebugFileTrace*, DebugFileTrace_instance);
CkpvAccess(DebugFileTrace_instance) = 0;
#endif
}
void initBigSimTrace(int outputParams, int _outputTiming)
{
CkpvInitialize(int, outputParameters);
CkpvAccess(outputParameters) = outputParams;
bgTraceCounter = 0;
#ifdef CMK_BIGSIM_CHARM
if (!BgIsReplay()) outputTiming = 0;
outputTiming = _outputTiming;
#endif
CkpvInitialize(bool, insideTraceBracket);
CkpvAccess(insideTraceBracket) = false;
CkpvInitialize(double, start_time);
CkpvInitialize(double, end_time);
CkpvInitialize(FILE*, bgfp);
CkpvAccess(bgfp) = NULL;
#ifdef CMK_BIGSIM_CHARM
// for bigsim emulation, write to files, one for each processor
// always write immediately, instead of store and dump at the end
if (!outputTiming) {
char fname[128];
const char *subdir = "params";
CmiMkdir(subdir);
sprintf(fname, "%s/param.%d", subdir, CkMyPe());
CkpvAccess(bgfp) = fopen(fname, "w");
if (CkpvAccess(bgfp) == NULL)
CmiAbort("Failed to generated trace param file!");
}
#endif
// for Mambo simulation, write to screen for now
// CkpvAccess(bgfp) = stdout;
if (CkpvAccess(outputParameters)) {
CkpvInitialize(StringPool, eventsPool);
if (CkMyPe()==0) CmiPrintf("outputParameters enabled!\n");
#ifdef CMK_BIGSIM_CHARM
BgRegisterUserTracingFunction(finalizeBigSimTrace);
#endif
}
#ifdef BIG_SIM_PAPI
CkPrintf("PAPI: number of available counters: %d\n", PAPI_num_counters());
CkAssert(PAPI_num_counters() >= 0);
#endif
}
void CpdFinishInitialization() {
CkpvInitialize(int, skipBreakpoint);
CkpvAccess(skipBreakpoint) = 0;
CkpvInitialize(DebugEntryTable, _debugEntryTable);
CkpvAccess(_debugEntryTable).resize(_entryTable.size());
}
/**
This is the main charm setup routine. It's called
on all processors after Converse initialization.
This routine gets passed to Converse from "main.C".
The main purpose of this routine is to set up the objects
and Ckpv's used during a regular Charm run. See the comment
at the top of the file for overall flow.
*/
void _initCharm(int unused_argc, char **argv)
{
int inCommThread = (CmiMyRank() == CmiMyNodeSize());
DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer()));
CkpvInitialize(size_t *, _offsets);
CkpvAccess(_offsets) = new size_t[32];
CkpvInitialize(PtrQ*,_buffQ);
CkpvInitialize(PtrVec*,_bocInitVec);
CkpvInitialize(void*, _currentChare);
CkpvInitialize(int, _currentChareType);
CkpvInitialize(CkGroupID, _currentGroup);
CkpvInitialize(void *, _currentNodeGroupObj);
CkpvInitialize(CkGroupID, _currentGroupRednMgr);
CkpvInitialize(GroupTable*, _groupTable);
CkpvInitialize(GroupIDTable*, _groupIDTable);
CkpvInitialize(CmiImmediateLockType, _groupTableImmLock);
CkpvInitialize(bool, _destroyingNodeGroup);
CkpvAccess(_destroyingNodeGroup) = false;
CkpvInitialize(UInt, _numGroups);
CkpvInitialize(int, _numInitsRecd);
CkpvInitialize(int, _initdone);
CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv;
CkpvInitialize(MsgPool*, _msgPool);
CkpvInitialize(CkCoreState *, _coreState);
/*
Added for evacuation-sayantan
*/
#ifndef __BIGSIM__
CpvInitialize(char *,_validProcessors);
#endif
CkpvInitialize(char ,startedEvac);
CpvInitialize(int,serializer);
_initChareTables(); // for checkpointable plain chares
CksvInitialize(UInt, _numNodeGroups);
CksvInitialize(GroupTable*, _nodeGroupTable);
CksvInitialize(GroupIDTable, _nodeGroupIDTable);
CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock);
CksvInitialize(CmiNodeLock, _nodeLock);
CksvInitialize(PtrVec*,_nodeBocInitVec);
CksvInitialize(UInt,_numInitNodeMsgs);
CkpvInitialize(int,_charmEpoch);
CkpvAccess(_charmEpoch)=0;
CksvInitialize(int, _triggersSent);
CksvAccess(_triggersSent) = 0;
CkpvInitialize(_CkOutStream*, _ckout);
CkpvInitialize(_CkErrStream*, _ckerr);
CkpvInitialize(Stats*, _myStats);
CkpvAccess(_groupIDTable) = new GroupIDTable(0);
CkpvAccess(_groupTable) = new GroupTable;
CkpvAccess(_groupTable)->init();
CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock();
CkpvAccess(_numGroups) = 1; // make 0 an invalid group number
CkpvAccess(_buffQ) = new PtrQ();
CkpvAccess(_bocInitVec) = new PtrVec();
CkpvAccess(_currentNodeGroupObj) = NULL;
if(CkMyRank()==0)
{
CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number
CksvAccess(_numInitNodeMsgs) = 0;
CksvAccess(_nodeLock) = CmiCreateLock();
CksvAccess(_nodeGroupTable) = new GroupTable();
CksvAccess(_nodeGroupTable)->init();
CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock();
CksvAccess(_nodeBocInitVec) = new PtrVec();
}
CkCallbackInit();
CmiNodeAllBarrier();
#if ! CMK_BIGSIM_CHARM
initQd(argv); // bigsim calls it in ConverseCommonInit
#endif
CkpvAccess(_coreState)=new CkCoreState();
CkpvAccess(_numInitsRecd) = 0;
CkpvAccess(_initdone) = 0;
CkpvAccess(_ckout) = new _CkOutStream();
CkpvAccess(_ckerr) = new _CkErrStream();
_charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler);
_initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
_roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler);
_exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler);
//added for interoperabilitY
_libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler);
_bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
#ifdef __BIGSIM__
if(BgNodeRank()==0)
#endif
_infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn);
_triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler);
_ckModuleInit();
CldRegisterEstimator((CldEstimator)_charmLoadEstimator);
_futuresModuleInit(); // part of futures implementation is a converse module
_loadbalancerInit();
_metabalancerInit();
#if CMK_MEM_CHECKPOINT
init_memcheckpt(argv);
#endif
initCharmProjections();
#if CMK_TRACE_IN_CHARM
// initialize trace module in ck
traceCharmInit(argv);
#endif
CkpvInitialize(int, envelopeEventID);
CkpvAccess(envelopeEventID) = 0;
CkMessageWatcherInit(argv,CkpvAccess(_coreState));
/**
The rank-0 processor of each node calls the
translator-generated "_register" routines.
_register routines call the charm.h "CkRegister*" routines,
which record function pointers and class information for
all Charm entities, like Chares, Arrays, and readonlies.
There's one _register routine generated for each
.ci file. _register routines *must* be called in the
same order on every node, and *must not* be called by
multiple threads simultaniously.
*/
#ifdef __BIGSIM__
if(BgNodeRank()==0)
#else
if(CkMyRank()==0)
#endif
{
SDAG::registerPUPables();
CmiArgGroup("Charm++",NULL);
_parseCommandLineOpts(argv);
_registerInit();
CkRegisterMsg("System", 0, 0, CkFreeMsg, sizeof(int));
CkRegisterChareInCharm(CkRegisterChare("null", 0, TypeChare));
CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare), TypeChare);
CkRegisterChareInCharm(CkIndex_Chare::__idx);
CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group), TypeGroup);
CkRegisterChareInCharm(CkIndex_Group::__idx);
CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC);
/**
These _register calls are for the built-in
Charm .ci files, like arrays and load balancing.
If you add a .ci file to charm, you'll have to
add a call to the _register routine here, or make
your library into a "-module".
*/
_registerCkFutures();
_registerCkArray();
_registerLBDatabase();
_registerMetaBalancer();
_registerCkCallback();
_registertempo();
_registerwaitqd();
_registerCkCheckpoint();
#if CMK_MEM_CHECKPOINT
_registerCkMemCheckpoint();
#endif
/*
Setup Control Point Automatic Tuning Framework.
By default it is enabled as a part of charm,
however it won't enable its tracing module
unless a +CPEnableMeasurements command line argument
is specified. See trace-common.C for more info
Thus there should be no noticable overhead to
always having the control point framework linked
in.
*/
#if CMK_WITH_CONTROLPOINT
_registerPathHistory();
_registerControlPoints();
_registerTraceControlPoints();
#endif
/**
CkRegisterMainModule is generated by the (unique)
"mainmodule" .ci file. It will include calls to
register all the .ci files.
*/
CkRegisterMainModule();
/**
_registerExternalModules is actually generated by
charmc at link time (as "moduleinit<pid>.C").
This generated routine calls the _register functions
for the .ci files of libraries linked using "-module".
This funny initialization is most useful for AMPI/FEM
programs, which don't have a .ci file and hence have
no other way to control the _register process.
*/
_registerExternalModules(argv);
_registerDone();
}
/* The following will happen on every virtual processor in BigEmulator, not just on once per real processor */
if (CkMyRank() == 0) {
CpdBreakPointInit();
}
CmiNodeAllBarrier();
// Execute the initcalls registered in modules
_initCallTable.enumerateInitCalls();
#if CMK_CHARMDEBUG
CpdFinishInitialization();
#endif
//CmiNodeAllBarrier();
CkpvAccess(_myStats) = new Stats();
CkpvAccess(_msgPool) = new MsgPool();
CmiNodeAllBarrier();
#if !(__FAULT__)
CmiBarrier();
CmiBarrier();
CmiBarrier();
#endif
#if CMK_SMP_TRACE_COMMTHREAD
_TRACE_BEGIN_COMPUTATION();
#else
if (!inCommThread) {
_TRACE_BEGIN_COMPUTATION();
}
#endif
#ifdef ADAPT_SCHED_MEM
if(CkMyRank()==0){
memCriticalEntries = new int[numMemCriticalEntries];
int memcnt=0;
for(int i=0; i<_entryTable.size(); i++){
if(_entryTable[i]->isMemCritical){
memCriticalEntries[memcnt++] = i;
}
}
}
#endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
_messageLoggingInit();
#endif
#ifndef __BIGSIM__
/*
FAULT_EVAC
*/
CpvAccess(_validProcessors) = new char[CkNumPes()];
for(int vProc=0;vProc<CkNumPes();vProc++){
CpvAccess(_validProcessors)[vProc]=1;
}
_ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast);
_ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac);
#endif
CkpvAccess(startedEvac) = 0;
CpvAccess(serializer) = 0;
evacuate = 0;
CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CcdCallOnCondition(CcdSIGUSR2,(CcdVoidFn)CkMlogRestart,0);
#endif
if(_raiseEvac){
processRaiseEvacFile(_raiseEvacFile);
/*
if(CkMyPe() == 2){
// CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
}
if(CkMyPe() == 3){
CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
}*/
}
if (CkMyRank() == 0) {
TopoManager_init();
}
CmiNodeAllBarrier();
if (!_replaySystem) {
CkFtFn faultFunc_restart = CkRestartMain;
if (faultFunc == NULL || faultFunc == faultFunc_restart) { // this is not restart from memory
// these two are blocking calls for non-bigsim
#if ! CMK_BIGSIM_CHARM
CmiInitCPUAffinity(argv);
CmiInitMemAffinity(argv);
#endif
}
CmiInitCPUTopology(argv);
#if CMK_SHARED_VARS_POSIX_THREADS_SMP
if (CmiCpuTopologyEnabled()) {
int *pelist;
int num;
CmiGetPesOnPhysicalNode(0, &pelist, &num);
#if !CMK_MULTICORE && !CMK_SMP_NO_COMMTHD
// Count communication threads, if present
// XXX: Assuming uniformity of node size here
num += num/CmiMyNodeSize();
#endif
if (!_Cmi_forceSpinOnIdle && num > CmiNumCores())
{
if (CmiMyPe() == 0)
CmiPrintf("\nCharm++> Warning: the number of SMP threads (%d) is greater than the number of physical cores (%d), so threads will sleep while idling. Use +CmiSpinOnIdle or +CmiSleepOnIdle to control this directly.\n\n", num, CmiNumCores());
CmiLock(CksvAccess(_nodeLock));
if (! _Cmi_sleepOnIdle) _Cmi_sleepOnIdle = 1;
CmiUnlock(CksvAccess(_nodeLock));
}
}
#endif
}
if(CmiMyPe() == 0) {
char *topoFilename;
if(CmiGetArgStringDesc(argv,"+printTopo",&topoFilename,"topo file name"))
{
std::stringstream sstm;
sstm << topoFilename << "." << CmiMyPartition();
std::string result = sstm.str();
FILE *fp;
fp = fopen(result.c_str(), "w");
if (fp == NULL) {
CkPrintf("Error opening %s file, writing to stdout\n", topoFilename);
fp = stdout;
}
TopoManager_printAllocation(fp);
fclose(fp);
}
}
#if CMK_USE_PXSHM && ( CMK_CRAYXE || CMK_CRAYXC ) && CMK_SMP
// for SMP on Cray XE6 (hopper) it seems pxshm has to be initialized
// again after cpuaffinity is done
if (CkMyRank() == 0) {
CmiInitPxshm(argv);
}
CmiNodeAllBarrier();
#endif
//CldCallback();
#if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG
// Register the BG handler for CCS. Notice that this is put into a variable shared by
// the whole real processor. This because converse needs to find it. We check that all
// virtual processors register the same index for this handler.
CpdBgInit();
#endif
if (faultFunc) {
#if CMK_WITH_STATS
if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()];
#endif
if (!inCommThread) {
CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
msg->argc = CmiGetArgc(argv);
msg->argv = argv;
faultFunc(_restartDir, msg);
CkFreeMsg(msg);
}
}else if(CkMyPe()==0){
#if CMK_WITH_STATS
_allStats = new Stats*[CkNumPes()];
#endif
register size_t i, nMains=_mainTable.size();
for(i=0;i<nMains;i++) /* Create all mainchares */
{
register int size = _chareTable[_mainTable[i]->chareIdx]->size;
register void *obj = malloc(size);
_MEMCHECK(obj);
_mainTable[i]->setObj(obj);
CkpvAccess(_currentChare) = obj;
CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx;
register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
msg->argc = CmiGetArgc(argv);
msg->argv = argv;
_entryTable[_mainTable[i]->entryIdx]->call(msg, obj);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CpvAccess(_currentObj) = (Chare *)obj;
#endif
}
_mainDone = 1;
_STATS_RECORD_CREATE_CHARE_N(nMains);
_STATS_RECORD_PROCESS_CHARE_N(nMains);
for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */
{
register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg));
if(roMsg==0)
continue;
//Pack the message and send it to all other processors
register envelope *env = UsrToEnv(roMsg);
env->setSrcPe(CkMyPe());
env->setMsgtype(ROMsgMsg);
env->setRoIdx(i);
CmiSetHandler(env, _initHandlerIdx);
CkPackMessage(&env);
CmiSyncBroadcast(env->getTotalsize(), (char *)env);
CpvAccess(_qd)->create(CkNumPes()-1);
//For processor 0, unpack and re-set the global
CkUnpackMessage(&env);
_processROMsgMsg(env);
_numInitMsgs++;
}
//Determine the size of the RODataMessage
PUP::sizer ps;
for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
//Allocate and fill out the RODataMessage
envelope *env = _allocEnv(RODataMsg, ps.size());
PUP::toMem pp((char *)EnvToUsr(env));
for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
env->setCount(++_numInitMsgs);
env->setSrcPe(CkMyPe());
CmiSetHandler(env, _initHandlerIdx);
DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize()));
CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
CpvAccess(_qd)->create(CkNumPes()-1);
_initDone();
}
DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread));
// when I am a communication thread, I don't participate initDone.
if (inCommThread) {
CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler,
CkpvAccess(_coreState));
CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler
,
CkpvAccess(_coreState));
_processBufferedMsgs();
}
#if CMK_CHARMDEBUG
// Should not use CpdFreeze inside a thread (since this processor is really a user-level thread)
if (CpvAccess(cpdSuspendStartup))
{
//CmiPrintf("In Parallel Debugging mode .....\n");
CpdFreeze();
}
#endif
#if __FAULT__
if(killFlag){
readKillFile();
}
#endif
}
// called from init.C
void _loadbalancerInit()
{
CkpvInitialize(int, lbdatabaseInited);
CkpvAccess(lbdatabaseInited) = 0;
CkpvInitialize(int, numLoadBalancers);
CkpvAccess(numLoadBalancers) = 0;
CkpvInitialize(int, hasNullLB);
CkpvAccess(hasNullLB) = 0;
CkpvInitialize(LBUserDataLayout, lbobjdatalayout);
CkpvInitialize(int, _lb_obj_index);
CkpvAccess(_lb_obj_index) = -1;
char **argv = CkGetArgv();
char *balancer = NULL;
CmiArgGroup("Charm++","Load Balancer");
while (CmiGetArgStringDesc(argv, "+balancer", &balancer, "Use this load balancer")) {
if (CkMyRank() == 0)
lbRegistry.addRuntimeBalancer(balancer); /* lbRegistry is a static */
}
// set up init value for LBPeriod time in seconds
// it can also be set by calling LDSetLBPeriod()
CmiGetArgDoubleDesc(argv,"+LBPeriod", &_lb_args.lbperiod(),"the minimum time period in seconds allowed for two consecutive automatic load balancing");
_lb_args.loop() = CmiGetArgFlagDesc(argv, "+LBLoop", "Use multiple load balancing strategies in loop");
// now called in cldb.c: CldModuleGeneralInit()
// registerLBTopos();
CmiGetArgStringDesc(argv, "+LBTopo", &_lbtopo, "define load balancing topology");
//Read the K parameter for RefineKLB
CmiGetArgIntDesc(argv, "+LBNumMoves", &_lb_args.percentMovesAllowed() , "Percentage of chares to be moved (used by RefineKLB) [0-100]");
/**************** FUTURE PREDICTOR ****************/
_lb_predict = CmiGetArgFlagDesc(argv, "+LBPredictor", "Turn on LB future predictor");
CmiGetArgIntDesc(argv, "+LBPredictorDelay", &_lb_predict_delay, "Number of balance steps before learning a model");
CmiGetArgIntDesc(argv, "+LBPredictorWindow", &_lb_predict_window, "Number of steps to use to learn a model");
if (_lb_predict_window < _lb_predict_delay) {
CmiPrintf("LB> [%d] Argument LBPredictorWindow (%d) less than LBPredictorDelay (%d) , fixing\n", CkMyPe(), _lb_predict_window, _lb_predict_delay);
_lb_predict_delay = _lb_predict_window;
}
/******************* SIMULATION *******************/
// get the step number at which to dump the LB database
CmiGetArgIntDesc(argv, "+LBVersion", &_lb_args.lbversion(), "LB database file version number");
CmiGetArgIntDesc(argv, "+LBCentPE", &_lb_args.central_pe(), "CentralLB processor");
int _lb_dump_activated = 0;
if (CmiGetArgIntDesc(argv, "+LBDump", &LBSimulation::dumpStep, "Dump the LB state from this step"))
_lb_dump_activated = 1;
if (_lb_dump_activated && LBSimulation::dumpStep < 0) {
CmiPrintf("LB> Argument LBDump (%d) negative, setting to 0\n",LBSimulation::dumpStep);
LBSimulation::dumpStep = 0;
}
CmiGetArgIntDesc(argv, "+LBDumpSteps", &LBSimulation::dumpStepSize, "Dump the LB state for this amount of steps");
if (LBSimulation::dumpStepSize <= 0) {
CmiPrintf("LB> Argument LBDumpSteps (%d) too small, setting to 1\n",LBSimulation::dumpStepSize);
LBSimulation::dumpStepSize = 1;
}
CmiGetArgStringDesc(argv, "+LBDumpFile", &LBSimulation::dumpFile, "Set the LB state file name");
// get the simulation flag and number. Now the flag can also be avoided by the presence of the number
LBSimulation::doSimulation = CmiGetArgIntDesc(argv, "+LBSim", &LBSimulation::simStep, "Read LB state from LBDumpFile since this step");
// check for stupid LBSim parameter
if (LBSimulation::doSimulation && LBSimulation::simStep < 0) {
CmiPrintf("LB> Argument LBSim (%d) invalid, should be >= 0\n");
CkExit();
return;
}
CmiGetArgIntDesc(argv, "+LBSimSteps", &LBSimulation::simStepSize, "Read LB state for this number of steps");
if (LBSimulation::simStepSize <= 0) {
CmiPrintf("LB> Argument LBSimSteps (%d) too small, setting to 1\n",LBSimulation::simStepSize);
LBSimulation::simStepSize = 1;
}
LBSimulation::simProcs = 0;
CmiGetArgIntDesc(argv, "+LBSimProcs", &LBSimulation::simProcs, "Number of target processors.");
LBSimulation::showDecisionsOnly =
CmiGetArgFlagDesc(argv, "+LBShowDecisions",
"Write to File: Load Balancing Object to Processor Map decisions during LB Simulation");
// force a global barrier after migration done
_lb_args.syncResume() = CmiGetArgFlagDesc(argv, "+LBSyncResume",
"LB performs a barrier after migration is finished");
// both +LBDebug and +LBDebug level should work
if (!CmiGetArgIntDesc(argv, "+LBDebug", &_lb_args.debug(),
"Turn on LB debugging printouts"))
_lb_args.debug() = CmiGetArgFlagDesc(argv, "+LBDebug",
"Turn on LB debugging printouts");
// getting the size of the team with +teamSize
if (!CmiGetArgIntDesc(argv, "+teamSize", &_lb_args.teamSize(),
"Team size"))
_lb_args.teamSize() = 1;
// ask to print summary/quality of load balancer
_lb_args.printSummary() = CmiGetArgFlagDesc(argv, "+LBPrintSummary",
"Print load balancing result summary");
// to ignore baclground load
_lb_args.ignoreBgLoad() = CmiGetArgFlagDesc(argv, "+LBNoBackground",
"Load balancer ignores the background load.");
#ifdef __BIGSIM__
_lb_args.ignoreBgLoad() = 1;
#endif
_lb_args.migObjOnly() = CmiGetArgFlagDesc(argv, "+LBObjOnly",
"Only load balancing migratable objects, ignoring all others.");
if (_lb_args.migObjOnly()) _lb_args.ignoreBgLoad() = 1;
// assume all CPUs are identical
_lb_args.testPeSpeed() = CmiGetArgFlagDesc(argv, "+LBTestPESpeed",
"Load balancer test all CPUs speed.");
_lb_args.samePeSpeed() = CmiGetArgFlagDesc(argv, "+LBSameCpus",
"Load balancer assumes all CPUs are of same speed.");
if (!_lb_args.testPeSpeed()) _lb_args.samePeSpeed() = 1;
_lb_args.useCpuTime() = CmiGetArgFlagDesc(argv, "+LBUseCpuTime",
"Load balancer uses CPU time instead of wallclock time.");
// turn instrumentation off at startup
_lb_args.statsOn() = !CmiGetArgFlagDesc(argv, "+LBOff",
"Turn load balancer instrumentation off");
// turn instrumentation of communicatin off at startup
_lb_args.traceComm() = !CmiGetArgFlagDesc(argv, "+LBCommOff",
"Turn load balancer instrumentation of communication off");
// turn on MetaBalancer if set
_lb_args.metaLbOn() = CmiGetArgFlagDesc(argv, "+MetaLB",
"Turn on MetaBalancer");
// set alpha and beta
_lb_args.alpha() = PER_MESSAGE_SEND_OVERHEAD_DEFAULT;
_lb_args.beta() = PER_BYTE_SEND_OVERHEAD_DEFAULT;
CmiGetArgDoubleDesc(argv,"+LBAlpha", &_lb_args.alpha(),
"per message send overhead");
CmiGetArgDoubleDesc(argv,"+LBBeta", &_lb_args.beta(),
"per byte send overhead");
if (CkMyPe() == 0) {
if (_lb_args.debug()) {
CmiPrintf("CharmLB> Verbose level %d, load balancing period: %g seconds\n", _lb_args.debug(), _lb_args.lbperiod());
}
if (_lb_args.debug() > 1) {
CmiPrintf("CharmLB> Topology %s alpha: %es beta: %es.\n", _lbtopo, _lb_args.alpha(), _lb_args.beta());
}
if (_lb_args.printSummary())
CmiPrintf("CharmLB> Load balancer print summary of load balancing result.\n");
if (_lb_args.ignoreBgLoad())
CmiPrintf("CharmLB> Load balancer ignores processor background load.\n");
if (_lb_args.samePeSpeed())
CmiPrintf("CharmLB> Load balancer assumes all CPUs are same.\n");
if (_lb_args.useCpuTime())
CmiPrintf("CharmLB> Load balancer uses CPU time instead of wallclock time.\n");
if (LBSimulation::doSimulation)
CmiPrintf("CharmLB> Load balancer running in simulation mode on file '%s' version %d.\n", LBSimulation::dumpFile, _lb_args.lbversion());
if (_lb_args.statsOn()==0)
CkPrintf("CharmLB> Load balancing instrumentation is off.\n");
if (_lb_args.traceComm()==0)
CkPrintf("CharmLB> Load balancing instrumentation for communication is off.\n");
if (_lb_args.migObjOnly())
CkPrintf("LB> Load balancing strategy ignores non-migratable objects.\n");
}
}
