/**
* This handler is used only during initialization. It receives messages from
* processor zero regarding Readonly Data (in one single message), Readonly Messages,
* Groups, and Nodegroups.
* The Readonly Data message also contains the total number of messages expected
* during the initialization phase.
* For Groups and Nodegroups, only messages with epoch=0 (meaning created from within
* a mainchare) are buffered for special creation, the other messages are buffered
* together with all the other regular messages by _bufferHandler (and will be flushed
* after all the initialization messages have been processed).
*/
static void _initHandler(void *msg, CkCoreState *ck)
{
CkAssert(CkMyPe()!=0);
register envelope *env = (envelope *) msg;
if (ck->watcher!=NULL) {
if (!ck->watcher->processMessage(&env,ck)) return;
}
switch (env->getMsgtype()) {
case BocInitMsg:
if (env->getGroupEpoch()==0) {
CkpvAccess(_numInitsRecd)++;
// _processBocInitMsg already handles QD
//CpvAccess(_qd)->process();
CkpvAccess(_bocInitVec)->insert(env->getGroupNum().idx, env);
} else _bufferHandler(msg);
break;
case NodeBocInitMsg:
if (env->getGroupEpoch()==0) {
CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
CksvAccess(_numInitNodeMsgs)++;
CksvAccess(_nodeBocInitVec)->insert(env->getGroupNum().idx, env);
CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
CpvAccess(_qd)->process();
} else _bufferHandler(msg);
break;
case ROMsgMsg:
CkpvAccess(_numInitsRecd)++;
CpvAccess(_qd)->process();
if(env->isPacked()) CkUnpackMessage(&env);
_processROMsgMsg(env);
break;
case RODataMsg:
CkpvAccess(_numInitsRecd)++;
CpvAccess(_qd)->process();
_numExpectInitMsgs = env->getCount();
_processRODataMsg(env);
break;
default:
CmiAbort("Internal Error: Unknown-msg-type. Contact Developers.\n");
}
DEBUGF(("[%d,%.6lf] _numExpectInitMsgs %d CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs) %d+%d\n",CmiMyPe(),CmiWallTimer(),_numExpectInitMsgs,CkpvAccess(_numInitsRecd),CksvAccess(_numInitNodeMsgs)));
if(_numExpectInitMsgs&&(CkpvAccess(_numInitsRecd)+CksvAccess(_numInitNodeMsgs)==_numExpectInitMsgs)) {
_initDone();
}
}
/**
* Converse handler receiving a signal from another processors in the same node.
* (On _sendTrigger there is the explanation of why this is necessary)
* Simply check if with the NodeGroup processed by another processor we reached
* the expected count. Notice that it can only be called before _initDone: after
* _initDone, a message destined for this handler will go instead to the _discardHandler.
*/
static void _triggerHandler(envelope *env)
{
if (_numExpectInitMsgs && CkpvAccess(_numInitsRecd) + CksvAccess(_numInitNodeMsgs) == _numExpectInitMsgs)
{
DEBUGF(("Calling Init Done from _triggerHandler\n"));
_initDone();
}
if (env!=NULL) CmiFree(env);
}
/**
* This function is used to send other processors on the same node a signal so
* they can check if their _initDone can be called: the reason for this is that
* the check at the end of _initHandler can fail due to a missing message containing
* a Nodegroup creation. When that message arrives only one processor will receive
* it, and thus if no notification is sent to the other processors in the node, they
* will never proceed.
*/
static void _sendTriggers(void)
{
int i, num, first;
CmiImmediateLock(CksvAccess(_nodeGroupTableImmLock));
if (CksvAccess(_triggersSent) == 0)
{
CksvAccess(_triggersSent)++;
num = CmiMyNodeSize();
register envelope *env = _allocEnv(RODataMsg); // Notice that the type here is irrelevant
env->setSrcPe(CkMyPe());
CmiSetHandler(env, _triggerHandlerIdx);
first = CmiNodeFirst(CmiMyNode());
for (i=0; i < num; i++)
if(first+i != CkMyPe())
CmiSyncSend(first+i, env->getTotalsize(), (char *)env);
CmiFree(env);
}
CmiImmediateUnlock(CksvAccess(_nodeGroupTableImmLock));
}
/**
* Create all nodegroups in this node (called only by rank zero, and never on node zero).
* Notice that only nodegroups created in mainchares are processed here;
* nodegroups created later are processed as regular messages.
*/
static inline void _processBufferedNodeBocInits(void)
{
CkCoreState *ck = CkpvAccess(_coreState);
register int i = 0;
PtrVec &inits=*CksvAccess(_nodeBocInitVec);
register int len = inits.size();
for(i=1; i<len; i++) {
envelope *env = inits[i];
if(env==0) CkAbort("_processBufferedNodeBocInits: empty message");
if(env->isPacked())
CkUnpackMessage(&env);
_processNodeBocInitMsg(ck,env);
}
delete &inits;
}
/**
* This function (not a handler) is called once and only once per processor.
* It signals the processor that the initialization is done and regular messages
* can be processed.
*
* On processor zero it is called by _initCharm, on all other processors either
* by _initHandler or _triggerHandler (cannot be both).
* When fault-tolerance is active, it is called by the fault-tolerance scheme itself.
*/
void _initDone(void)
{
if (CkpvAccess(_initdone)) return;
CkpvAccess(_initdone) ++;
DEBUGF(("[%d] _initDone.\n", CkMyPe()));
if (!CksvAccess(_triggersSent)) _sendTriggers();
CkNumberHandler(_triggerHandlerIdx, (CmiHandler)_discardHandler);
CmiNodeBarrier();
if(CkMyRank() == 0) {
_processBufferedNodeBocInits();
}
CmiNodeBarrier(); // wait for all nodegroups to be created
_processBufferedBocInits();
DEBUGF(("Reached CmiNodeBarrier(), pe = %d, rank = %d\n", CkMyPe(), CkMyRank()));
CmiNodeBarrier();
DEBUGF(("Crossed CmiNodeBarrier(), pe = %d, rank = %d\n", CkMyPe(), CkMyRank()));
_processBufferedMsgs();
CkpvAccess(_charmEpoch)=1;
}
// handle NodeGroupTable and data
void CkPupNodeGroupData(PUP::er &p, CmiBool create)
{
int numNodeGroups, i;
if (!p.isUnpacking()) {
numNodeGroups = CksvAccess(_nodeGroupIDTable).size();
}
p|numNodeGroups;
if (p.isUnpacking()) {
if(CkMyPe()==0){ CksvAccess(_numNodeGroups) = numNodeGroups+1; }
else { CksvAccess(_numNodeGroups) = 1; }
}
if(CkMyPe() == 3)
CkPrintf("[%d] CkPupNodeGroupData %s: numNodeGroups = %d\n",CkMyPe(),p.typeString(),numNodeGroups);
GroupInfo *tmpInfo = new GroupInfo [numNodeGroups];
if (!p.isUnpacking()) {
for(i=0;i<numNodeGroups;i++) {
tmpInfo[i].gID = CksvAccess(_nodeGroupIDTable)[i];
TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(tmpInfo[i].gID);
tmpInfo[i].MigCtor = _chareTable[ent2.getcIdx()]->migCtor;
if(tmpInfo[i].MigCtor==-1) {
char buf[512];
sprintf(buf,"NodeGroup %s either need a migration constructor and\n\
declared as [migratable] in .ci to be able to checkpoint.",\
_chareTable[ent2.getcIdx()]->name);
CkAbort(buf);
}
}
}
for (i=0; i<numNodeGroups; i++) p|tmpInfo[i];
for (i=0;i<numNodeGroups;i++) {
CkGroupID gID = tmpInfo[i].gID;
if (p.isUnpacking()) {
//CksvAccess(_nodeGroupIDTable).push_back(gID);
int eIdx = tmpInfo[i].MigCtor;
void *m = CkAllocSysMsg();
envelope* env = UsrToEnv((CkMessage *)m);
if(create){
CkCreateLocalNodeGroup(gID, eIdx, env);
}
}
TableEntry ent2 = CksvAccess(_nodeGroupTable)->find(gID);
IrrGroup *obj = ent2.getObj();
obj->pup(p);
if(CkMyPe() == 3) CkPrintf("Nodegroup PUP'ed: gid = %d, name = %s\n",
obj->ckGetGroupID().idx,
_chareTable[ent2.getcIdx()]->name);
}
delete [] tmpInfo;
}
/**
This is the main charm setup routine. It's called
on all processors after Converse initialization.
This routine gets passed to Converse from "main.C".
The main purpose of this routine is to set up the objects
and Ckpv's used during a regular Charm run. See the comment
at the top of the file for overall flow.
*/
void _initCharm(int unused_argc, char **argv)
{
int inCommThread = (CmiMyRank() == CmiMyNodeSize());
DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer()));
CkpvInitialize(size_t *, _offsets);
CkpvAccess(_offsets) = new size_t[32];
CkpvInitialize(PtrQ*,_buffQ);
CkpvInitialize(PtrVec*,_bocInitVec);
CkpvInitialize(void*, _currentChare);
CkpvInitialize(int, _currentChareType);
CkpvInitialize(CkGroupID, _currentGroup);
CkpvInitialize(void *, _currentNodeGroupObj);
CkpvInitialize(CkGroupID, _currentGroupRednMgr);
CkpvInitialize(GroupTable*, _groupTable);
CkpvInitialize(GroupIDTable*, _groupIDTable);
CkpvInitialize(CmiImmediateLockType, _groupTableImmLock);
CkpvInitialize(bool, _destroyingNodeGroup);
CkpvAccess(_destroyingNodeGroup) = false;
CkpvInitialize(UInt, _numGroups);
CkpvInitialize(int, _numInitsRecd);
CkpvInitialize(int, _initdone);
CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv;
CkpvInitialize(MsgPool*, _msgPool);
CkpvInitialize(CkCoreState *, _coreState);
/*
Added for evacuation-sayantan
*/
#ifndef __BIGSIM__
CpvInitialize(char *,_validProcessors);
#endif
CkpvInitialize(char ,startedEvac);
CpvInitialize(int,serializer);
_initChareTables(); // for checkpointable plain chares
CksvInitialize(UInt, _numNodeGroups);
CksvInitialize(GroupTable*, _nodeGroupTable);
CksvInitialize(GroupIDTable, _nodeGroupIDTable);
CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock);
CksvInitialize(CmiNodeLock, _nodeLock);
CksvInitialize(PtrVec*,_nodeBocInitVec);
CksvInitialize(UInt,_numInitNodeMsgs);
CkpvInitialize(int,_charmEpoch);
CkpvAccess(_charmEpoch)=0;
CksvInitialize(int, _triggersSent);
CksvAccess(_triggersSent) = 0;
CkpvInitialize(_CkOutStream*, _ckout);
CkpvInitialize(_CkErrStream*, _ckerr);
CkpvInitialize(Stats*, _myStats);
CkpvAccess(_groupIDTable) = new GroupIDTable(0);
CkpvAccess(_groupTable) = new GroupTable;
CkpvAccess(_groupTable)->init();
CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock();
CkpvAccess(_numGroups) = 1; // make 0 an invalid group number
CkpvAccess(_buffQ) = new PtrQ();
CkpvAccess(_bocInitVec) = new PtrVec();
CkpvAccess(_currentNodeGroupObj) = NULL;
if(CkMyRank()==0)
{
CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number
CksvAccess(_numInitNodeMsgs) = 0;
CksvAccess(_nodeLock) = CmiCreateLock();
CksvAccess(_nodeGroupTable) = new GroupTable();
CksvAccess(_nodeGroupTable)->init();
CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock();
CksvAccess(_nodeBocInitVec) = new PtrVec();
}
CkCallbackInit();
CmiNodeAllBarrier();
#if ! CMK_BIGSIM_CHARM
initQd(argv); // bigsim calls it in ConverseCommonInit
#endif
CkpvAccess(_coreState)=new CkCoreState();
CkpvAccess(_numInitsRecd) = 0;
CkpvAccess(_initdone) = 0;
CkpvAccess(_ckout) = new _CkOutStream();
CkpvAccess(_ckerr) = new _CkErrStream();
_charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler);
_initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
_roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler);
_exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler);
//added for interoperabilitY
_libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler);
_bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
#ifdef __BIGSIM__
if(BgNodeRank()==0)
#endif
_infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn);
_triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler);
_ckModuleInit();
CldRegisterEstimator((CldEstimator)_charmLoadEstimator);
_futuresModuleInit(); // part of futures implementation is a converse module
_loadbalancerInit();
_metabalancerInit();
#if CMK_MEM_CHECKPOINT
init_memcheckpt(argv);
#endif
initCharmProjections();
#if CMK_TRACE_IN_CHARM
// initialize trace module in ck
traceCharmInit(argv);
#endif
CkpvInitialize(int, envelopeEventID);
CkpvAccess(envelopeEventID) = 0;
CkMessageWatcherInit(argv,CkpvAccess(_coreState));
/**
The rank-0 processor of each node calls the
translator-generated "_register" routines.
_register routines call the charm.h "CkRegister*" routines,
which record function pointers and class information for
all Charm entities, like Chares, Arrays, and readonlies.
There's one _register routine generated for each
.ci file. _register routines *must* be called in the
same order on every node, and *must not* be called by
multiple threads simultaniously.
*/
#ifdef __BIGSIM__
if(BgNodeRank()==0)
#else
if(CkMyRank()==0)
#endif
{
SDAG::registerPUPables();
CmiArgGroup("Charm++",NULL);
_parseCommandLineOpts(argv);
_registerInit();
CkRegisterMsg("System", 0, 0, CkFreeMsg, sizeof(int));
CkRegisterChareInCharm(CkRegisterChare("null", 0, TypeChare));
CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare), TypeChare);
CkRegisterChareInCharm(CkIndex_Chare::__idx);
CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group), TypeGroup);
CkRegisterChareInCharm(CkIndex_Group::__idx);
CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC);
/**
These _register calls are for the built-in
Charm .ci files, like arrays and load balancing.
If you add a .ci file to charm, you'll have to
add a call to the _register routine here, or make
your library into a "-module".
*/
_registerCkFutures();
_registerCkArray();
_registerLBDatabase();
_registerMetaBalancer();
_registerCkCallback();
_registertempo();
_registerwaitqd();
_registerCkCheckpoint();
#if CMK_MEM_CHECKPOINT
_registerCkMemCheckpoint();
#endif
/*
Setup Control Point Automatic Tuning Framework.
By default it is enabled as a part of charm,
however it won't enable its tracing module
unless a +CPEnableMeasurements command line argument
is specified. See trace-common.C for more info
Thus there should be no noticable overhead to
always having the control point framework linked
in.
*/
#if CMK_WITH_CONTROLPOINT
_registerPathHistory();
_registerControlPoints();
_registerTraceControlPoints();
#endif
/**
CkRegisterMainModule is generated by the (unique)
"mainmodule" .ci file. It will include calls to
register all the .ci files.
*/
CkRegisterMainModule();
/**
_registerExternalModules is actually generated by
charmc at link time (as "moduleinit<pid>.C").
This generated routine calls the _register functions
for the .ci files of libraries linked using "-module".
This funny initialization is most useful for AMPI/FEM
programs, which don't have a .ci file and hence have
no other way to control the _register process.
*/
_registerExternalModules(argv);
_registerDone();
}
/* The following will happen on every virtual processor in BigEmulator, not just on once per real processor */
if (CkMyRank() == 0) {
CpdBreakPointInit();
}
CmiNodeAllBarrier();
// Execute the initcalls registered in modules
_initCallTable.enumerateInitCalls();
#if CMK_CHARMDEBUG
CpdFinishInitialization();
#endif
//CmiNodeAllBarrier();
CkpvAccess(_myStats) = new Stats();
CkpvAccess(_msgPool) = new MsgPool();
CmiNodeAllBarrier();
#if !(__FAULT__)
CmiBarrier();
CmiBarrier();
CmiBarrier();
#endif
#if CMK_SMP_TRACE_COMMTHREAD
_TRACE_BEGIN_COMPUTATION();
#else
if (!inCommThread) {
_TRACE_BEGIN_COMPUTATION();
}
#endif
#ifdef ADAPT_SCHED_MEM
if(CkMyRank()==0){
memCriticalEntries = new int[numMemCriticalEntries];
int memcnt=0;
for(int i=0; i<_entryTable.size(); i++){
if(_entryTable[i]->isMemCritical){
memCriticalEntries[memcnt++] = i;
}
}
}
#endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
_messageLoggingInit();
#endif
#ifndef __BIGSIM__
/*
FAULT_EVAC
*/
CpvAccess(_validProcessors) = new char[CkNumPes()];
for(int vProc=0;vProc<CkNumPes();vProc++){
CpvAccess(_validProcessors)[vProc]=1;
}
_ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast);
_ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac);
#endif
CkpvAccess(startedEvac) = 0;
CpvAccess(serializer) = 0;
evacuate = 0;
CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CcdCallOnCondition(CcdSIGUSR2,(CcdVoidFn)CkMlogRestart,0);
#endif
if(_raiseEvac){
processRaiseEvacFile(_raiseEvacFile);
/*
if(CkMyPe() == 2){
// CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
}
if(CkMyPe() == 3){
CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
}*/
}
if (CkMyRank() == 0) {
TopoManager_init();
}
CmiNodeAllBarrier();
if (!_replaySystem) {
CkFtFn faultFunc_restart = CkRestartMain;
if (faultFunc == NULL || faultFunc == faultFunc_restart) { // this is not restart from memory
// these two are blocking calls for non-bigsim
#if ! CMK_BIGSIM_CHARM
CmiInitCPUAffinity(argv);
CmiInitMemAffinity(argv);
#endif
}
CmiInitCPUTopology(argv);
#if CMK_SHARED_VARS_POSIX_THREADS_SMP
if (CmiCpuTopologyEnabled()) {
int *pelist;
int num;
CmiGetPesOnPhysicalNode(0, &pelist, &num);
#if !CMK_MULTICORE && !CMK_SMP_NO_COMMTHD
// Count communication threads, if present
// XXX: Assuming uniformity of node size here
num += num/CmiMyNodeSize();
#endif
if (!_Cmi_forceSpinOnIdle && num > CmiNumCores())
{
if (CmiMyPe() == 0)
CmiPrintf("\nCharm++> Warning: the number of SMP threads (%d) is greater than the number of physical cores (%d), so threads will sleep while idling. Use +CmiSpinOnIdle or +CmiSleepOnIdle to control this directly.\n\n", num, CmiNumCores());
CmiLock(CksvAccess(_nodeLock));
if (! _Cmi_sleepOnIdle) _Cmi_sleepOnIdle = 1;
CmiUnlock(CksvAccess(_nodeLock));
}
}
#endif
}
if(CmiMyPe() == 0) {
char *topoFilename;
if(CmiGetArgStringDesc(argv,"+printTopo",&topoFilename,"topo file name"))
{
std::stringstream sstm;
sstm << topoFilename << "." << CmiMyPartition();
std::string result = sstm.str();
FILE *fp;
fp = fopen(result.c_str(), "w");
if (fp == NULL) {
CkPrintf("Error opening %s file, writing to stdout\n", topoFilename);
fp = stdout;
}
TopoManager_printAllocation(fp);
fclose(fp);
}
}
#if CMK_USE_PXSHM && ( CMK_CRAYXE || CMK_CRAYXC ) && CMK_SMP
// for SMP on Cray XE6 (hopper) it seems pxshm has to be initialized
// again after cpuaffinity is done
if (CkMyRank() == 0) {
CmiInitPxshm(argv);
}
CmiNodeAllBarrier();
#endif
//CldCallback();
#if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG
// Register the BG handler for CCS. Notice that this is put into a variable shared by
// the whole real processor. This because converse needs to find it. We check that all
// virtual processors register the same index for this handler.
CpdBgInit();
#endif
if (faultFunc) {
#if CMK_WITH_STATS
if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()];
#endif
if (!inCommThread) {
CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
msg->argc = CmiGetArgc(argv);
msg->argv = argv;
faultFunc(_restartDir, msg);
CkFreeMsg(msg);
}
}else if(CkMyPe()==0){
#if CMK_WITH_STATS
_allStats = new Stats*[CkNumPes()];
#endif
register size_t i, nMains=_mainTable.size();
for(i=0;i<nMains;i++) /* Create all mainchares */
{
register int size = _chareTable[_mainTable[i]->chareIdx]->size;
register void *obj = malloc(size);
_MEMCHECK(obj);
_mainTable[i]->setObj(obj);
CkpvAccess(_currentChare) = obj;
CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx;
register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
msg->argc = CmiGetArgc(argv);
msg->argv = argv;
_entryTable[_mainTable[i]->entryIdx]->call(msg, obj);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CpvAccess(_currentObj) = (Chare *)obj;
#endif
}
_mainDone = 1;
_STATS_RECORD_CREATE_CHARE_N(nMains);
_STATS_RECORD_PROCESS_CHARE_N(nMains);
for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */
{
register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg));
if(roMsg==0)
continue;
//Pack the message and send it to all other processors
register envelope *env = UsrToEnv(roMsg);
env->setSrcPe(CkMyPe());
env->setMsgtype(ROMsgMsg);
env->setRoIdx(i);
CmiSetHandler(env, _initHandlerIdx);
CkPackMessage(&env);
CmiSyncBroadcast(env->getTotalsize(), (char *)env);
CpvAccess(_qd)->create(CkNumPes()-1);
//For processor 0, unpack and re-set the global
CkUnpackMessage(&env);
_processROMsgMsg(env);
_numInitMsgs++;
}
//Determine the size of the RODataMessage
PUP::sizer ps;
for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
//Allocate and fill out the RODataMessage
envelope *env = _allocEnv(RODataMsg, ps.size());
PUP::toMem pp((char *)EnvToUsr(env));
for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
env->setCount(++_numInitMsgs);
env->setSrcPe(CkMyPe());
CmiSetHandler(env, _initHandlerIdx);
DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize()));
CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
CpvAccess(_qd)->create(CkNumPes()-1);
_initDone();
}
DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread));
// when I am a communication thread, I don't participate initDone.
if (inCommThread) {
CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler,
CkpvAccess(_coreState));
CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler
,
CkpvAccess(_coreState));
_processBufferedMsgs();
}
#if CMK_CHARMDEBUG
// Should not use CpdFreeze inside a thread (since this processor is really a user-level thread)
if (CpvAccess(cpdSuspendStartup))
{
//CmiPrintf("In Parallel Debugging mode .....\n");
CpdFreeze();
}
#endif
#if __FAULT__
if(killFlag){
readKillFile();
}
#endif
}
