int CcsIsRemoteRequest(void) { return CpvAccess(ccsReq)!=NULL; }
/** * Register a callback function that will be triggered on the specified PE * *whenever* the specified condition is raised */ int CcdCallOnConditionKeepOnPE(int condnum, CcdVoidFn fnp, void *arg, int pe) { return append_elem(&(CpvAccess(conds).condcb_keep[condnum]), fnp, arg, pe); }
/** * Raise a condition causing all registered callbacks corresponding to * that condition to be triggered */ void CcdRaiseCondition(int condnum) { double curWallTime=CmiWallTimer(); call_cblist_remove(&(CpvAccess(conds).condcb[condnum]),curWallTime); call_cblist_keep(&(CpvAccess(conds).condcb_keep[condnum]),curWallTime); }
static void collectNumbers(ProcMsg *msg) { int npes; EmptyMsg emsg; if(CpvAccess(isSingle)) { CpvAccess(Time1) = CmiWallTimer() - CpvAccess(Time1); CpvAccess(seqPI) = 4.0 * msg->success / NTRIALS; CpvAccess(isSingle) = 0; CpvAccess(nreported) = 0; CpvAccess(success) = 0; msg->success = NTRIALS/CmiNumPes(); CmiSetHandler(msg, CpvAccess(trial_handler)); CmiSyncBroadcastAll(sizeof(ProcMsg), msg); CpvAccess(TimeN) = CmiWallTimer(); printf("if\n"); } else { printf("else\n"); CpvAccess(nreported)++; CpvAccess(success) += msg->success; if(CpvAccess(nreported)==CmiNumPes()) { CpvAccess(TimeN) = CmiWallTimer() - CpvAccess(TimeN); CpvAccess(parPI) = 4.0 * CpvAccess(success) / NTRIALS; npes = iround(CpvAccess(Time1)/CpvAccess(TimeN)); CmiPrintf("[proc] Tseq = %le seconds, Tpar = %le seconds\n", CpvAccess(Time1), CpvAccess(TimeN)); CmiPrintf("[proc] CmiNumPes() reported %d processors\n", CmiNumPes()); CmiPrintf("[proc] But actual number of processors is %d\n", npes); CmiPrintf("[proc] FYI, appox PI (seq) = %lf\n",CpvAccess(seqPI)); CmiPrintf("[proc] FYI, appox PI (par) = %lf\n",CpvAccess(parPI)); CmiSetHandler(&emsg, CpvAccess(ack_handler)); CmiSyncSend(0, sizeof(EmptyMsg), &emsg); printf("else if\n"); } } }
//Registering idle handlers void ApplIdleStart(void *, double start) { CpvAccess(IdleStartTime)= start; //CmiWallTimer(); return; }
//! clear all data collected for entry points void TraceCounter::traceClearEps() { CpvAccess(_logPool)->clearEps(); }
void pvmc_init_bufs(void) { int i; #ifdef PVM_DEBUG PRINTF("Pe(%d) tid=%d:%s:%d pvmc_init_bufs() initializing buffer array\n", MYPE(),pvm_mytid(),__FILE__,__LINE__); #endif CpvInitialize(pvmc_buffer*,pvmc_bufarray); CpvAccess(pvmc_bufarray)=MALLOC(sizeof(pvmc_buffer)*MAX_BUFFERS); if (CpvAccess(pvmc_bufarray)==NULL) { PRINTF("Pe(%d) tid=%d:%s:%d pvmc_init_bufs() can't alloc buffer array\n", MYPE(),pvm_mytid(),__FILE__,__LINE__); exit(1); } CpvInitialize(pvmc_buffer*,pvmc_freebufs); CpvAccess(pvmc_freebufs)=&(CpvAccess(pvmc_bufarray)[1]); /* throw away first bufid */ for(i=0;i<MAX_BUFFERS;i++) { CpvAccess(pvmc_bufarray)[i].bufid=i; CpvAccess(pvmc_bufarray)[i].bytes=0; CpvAccess(pvmc_bufarray)[i].tag=0; CpvAccess(pvmc_bufarray)[i].tid=-1; CpvAccess(pvmc_bufarray)[i].num_items=-1; CpvAccess(pvmc_bufarray)[i].refcount=0; CpvAccess(pvmc_bufarray)[i].first_item=(pvmc_item *)NULL; CpvAccess(pvmc_bufarray)[i].cur_item=(pvmc_item *)NULL; CpvAccess(pvmc_bufarray)[i].last_item=(pvmc_item *)NULL; if (i==MAX_BUFFERS-1) CpvAccess(pvmc_bufarray)[i].nxt_free=(pvmc_buffer *)NULL; else CpvAccess(pvmc_bufarray)[i].nxt_free=&(CpvAccess(pvmc_bufarray)[i+1]); CpvAccess(pvmc_bufarray)[i].data_buf=(char *)NULL; } CpvInitialize(int,pvmc_sbufid); CpvAccess(pvmc_sbufid) = 0; CpvInitialize(int,pvmc_rbufid); CpvAccess(pvmc_rbufid) = 0; }
static inline void _parseCommandLineOpts(char **argv) { if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown")) _STATS_ON(_printCS); if (CmiGetArgFlagDesc(argv,"+ss", "Print summary statistics at shutdown")) _STATS_ON(_printSS); if (CmiGetArgFlagDesc(argv,"+fifo", "Default to FIFO queuing")) _defaultQueueing = CK_QUEUEING_FIFO; if (CmiGetArgFlagDesc(argv,"+lifo", "Default to LIFO queuing")) _defaultQueueing = CK_QUEUEING_LIFO; if (CmiGetArgFlagDesc(argv,"+ififo", "Default to integer-prioritized FIFO queuing")) _defaultQueueing = CK_QUEUEING_IFIFO; if (CmiGetArgFlagDesc(argv,"+ilifo", "Default to integer-prioritized LIFO queuing")) _defaultQueueing = CK_QUEUEING_ILIFO; if (CmiGetArgFlagDesc(argv,"+bfifo", "Default to bitvector-prioritized FIFO queuing")) _defaultQueueing = CK_QUEUEING_BFIFO; if (CmiGetArgFlagDesc(argv,"+blifo", "Default to bitvector-prioritized LIFO queuing")) _defaultQueueing = CK_QUEUEING_BLIFO; if (CmiGetArgFlagDesc(argv,"+objq", "Default to use object queue for every obejct")) { #if CMK_OBJECT_QUEUE_AVAILABLE _defaultObjectQ = 1; if (CkMyPe()==0) CmiPrintf("Charm++> Create object queue for every Charm object.\n"); #else CmiAbort("Charm++> Object queue not enabled, recompile Charm++ with CMK_OBJECT_QUEUE_AVAILABLE defined to 1."); #endif } if(CmiGetArgString(argv,"+restart",&_restartDir)) faultFunc = CkRestartMain; #if __FAULT__ if (CmiGetArgIntDesc(argv,"+restartaftercrash",&CpvAccess(_curRestartPhase),"restarting this processor after a crash")){ # if CMK_MEM_CHECKPOINT faultFunc = CkMemRestart; # endif #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) faultFunc = CkMlogRestart; #endif CmiPrintf("[%d] Restarting after crash \n",CmiMyPe()); } #if CMK_MESSAGE_LOGGING // reading +ftc_disk flag if (CmiGetArgFlagDesc(argv, "+ftc_disk", "Disk Checkpointing")) { diskCkptFlag = 1; } #endif // reading the killFile if(CmiGetArgStringDesc(argv,"+killFile", &killFile,"Generates SIGKILL on specified processors")){ if(faultFunc == NULL){ //do not read the killfile if this is a restarting processor killFlag = 1; if(CmiMyPe() == 0){ printf("[%d] killFlag set to 1 for file %s\n",CkMyPe(),killFile); } } } #endif // shut down program in ring fashion to allow projections output w/o IO error if (CmiGetArgIntDesc(argv,"+ringexit",&_ringtoken, "Program exits in a ring fashion")) { _ringexit = 1; if (CkMyPe()==0) CkPrintf("Charm++> Program shutdown in token ring (%d).\n", _ringtoken); if (_ringtoken > CkNumPes()) _ringtoken = CkNumPes(); } /* FAULT_EVAC if the argument +raiseevac is present then cause faults */ if(CmiGetArgStringDesc(argv,"+raiseevac", &_raiseEvacFile,"Generates processor evacuation on random processors")){ _raiseEvac = 1; } #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) if(!CmiGetArgIntDesc(argv,"+teamSize",&teamSize,"Set the team size for message logging")){ teamSize = 1; } if(!CmiGetArgIntDesc(argv,"+chkptPeriod",&chkptPeriod,"Set the checkpoint period for the message logging fault tolerance algorithm in seconds")){ chkptPeriod = 100; } if(CmiGetArgIntDesc(argv,"+fastRecovery", ¶llelRecovery, "Parallel recovery with message logging protocol")){ fastRecovery = true; } #endif /* Anytime migration flag */ _isAnytimeMigration = true; if (CmiGetArgFlagDesc(argv,"+noAnytimeMigration","The program does not require support for anytime migration")) { _isAnytimeMigration = false; } _isNotifyChildInRed = true; if (CmiGetArgFlagDesc(argv,"+noNotifyChildInReduction","The program has at least one element per processor for each charm array created")) { _isNotifyChildInRed = false; } _isStaticInsertion = false; if (CmiGetArgFlagDesc(argv,"+staticInsertion","Array elements are only inserted at construction")) { _isStaticInsertion = true; } useNodeBlkMapping = false; if (CmiGetArgFlagDesc(argv,"+useNodeBlkMapping","Array elements are block-mapped in SMP-node level")) { useNodeBlkMapping = true; } #if ! CMK_WITH_CONTROLPOINT // Display a warning if charm++ wasn't compiled with control point support but user is expecting it if( CmiGetArgFlag(argv,"+CPSamplePeriod") || CmiGetArgFlag(argv,"+CPSamplePeriodMs") || CmiGetArgFlag(argv,"+CPSchemeRandom") || CmiGetArgFlag(argv,"+CPExhaustiveSearch") || CmiGetArgFlag(argv,"+CPAlwaysUseDefaults") || CmiGetArgFlag(argv,"+CPSimulAnneal") || CmiGetArgFlag(argv,"+CPCriticalPathPrio") || CmiGetArgFlag(argv,"+CPBestKnown") || CmiGetArgFlag(argv,"+CPSteering") || CmiGetArgFlag(argv,"+CPMemoryAware") || CmiGetArgFlag(argv,"+CPSimplex") || CmiGetArgFlag(argv,"+CPDivideConquer") || CmiGetArgFlag(argv,"+CPLDBPeriod") || CmiGetArgFlag(argv,"+CPLDBPeriodLinear") || CmiGetArgFlag(argv,"+CPLDBPeriodQuadratic") || CmiGetArgFlag(argv,"+CPLDBPeriodOptimal") || CmiGetArgFlag(argv,"+CPDefaultValues") || CmiGetArgFlag(argv,"+CPGatherAll") || CmiGetArgFlag(argv,"+CPGatherMemoryUsage") || CmiGetArgFlag(argv,"+CPGatherUtilization") || CmiGetArgFlag(argv,"+CPSaveData") || CmiGetArgFlag(argv,"+CPNoFilterData") || CmiGetArgFlag(argv,"+CPLoadData") || CmiGetArgFlag(argv,"+CPDataFilename") ) { CkAbort("You specified a control point command line argument, but compiled charm++ without control point support.\n"); } #endif }
static void _exitHandler(envelope *env) { DEBUGF(("exitHandler called on %d msgtype: %d\n", CkMyPe(), env->getMsgtype())); switch(env->getMsgtype()) { case StartExitMsg: CkAssert(CkMyPe()==0); if (!_CkExitFnVec.isEmpty()) { CkExitFn fn = _CkExitFnVec.deq(); fn(); break; } // else goto next case ExitMsg: CkAssert(CkMyPe()==0); if(_exitStarted) { CmiFree(env); return; } _exitStarted = 1; CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler); CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler); env->setMsgtype(ReqStatMsg); env->setSrcPe(CkMyPe()); // if exit in ring, instead of broadcasting, send in ring if (_ringexit){ DEBUGF(("[%d] Ring Exit \n",CkMyPe())); const int stride = CkNumPes()/_ringtoken; int pe = 0; while (pe<CkNumPes()) { CmiSyncSend(pe, env->getTotalsize(), (char *)env); pe += stride; } CmiFree(env); }else{ CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env); } break; case ReqStatMsg: #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) _messageLoggingExit(); #endif DEBUGF(("ReqStatMsg on %d\n", CkMyPe())); CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler); CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler); /*FAULT_EVAC*/ if(CmiNodeAlive(CkMyPe())){ #if CMK_WITH_STATS _sendStats(); #endif _mainDone = 1; // This is needed because the destructors for // readonly variables will be called when the program // exits. If the destructor is called while _mainDone // is 0, it will assume that the readonly variable was // declared locally. On all processors other than 0, // _mainDone is never set to 1 before the program exits. #if CMK_TRACE_ENABLED if (_ringexit) traceClose(); #endif } if (_ringexit) { int stride = CkNumPes()/_ringtoken; int pe = CkMyPe()+1; if (pe < CkNumPes() && pe % stride != 0) CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env); else CmiFree(env); } else CmiFree(env); //everyone exits here - there may be issues with leftover messages in the queue #if CMK_WITH_STATS if(CkMyPe()) #endif { DEBUGF(("[%d] Calling converse exit \n",CkMyPe())); ConverseExit(); if(CharmLibInterOperate) CpvAccess(interopExitFlag) = 1; } break; #if CMK_WITH_STATS case StatMsg: CkAssert(CkMyPe()==0); _allStats[env->getSrcPe()] = (Stats*) EnvToUsr(env); _numStatsRecd++; DEBUGF(("StatMsg on %d with %d\n", CkMyPe(), _numStatsRecd)); /*FAULT_EVAC*/ if(_numStatsRecd==CkNumValidPes()) { _printStats(); DEBUGF(("[%d] Calling converse exit \n",CkMyPe())); ConverseExit(); if(CharmLibInterOperate) CpvAccess(interopExitFlag) = 1; } break; #endif default: CmiAbort("Internal Error(_exitHandler): Unknown-msg-type. Contact Developers.\n"); } }
void CcsInit(char **argv) { CpvInitialize(CkHashtable_c, ccsTab); CpvAccess(ccsTab) = CkCreateHashtable_string(sizeof(CcsHandlerRec),5); CpvInitialize(CcsImplHeader *, ccsReq); CpvAccess(ccsReq) = NULL; _ccsHandlerIdx = CmiRegisterHandler((CmiHandler)req_fw_handler); #if CMK_BIGSIM_CHARM CpvInitialize(int, _bgCcsHandlerIdx); CpvAccess(_bgCcsHandlerIdx) = 0; CpvInitialize(int, _bgCcsAck); CpvAccess(_bgCcsAck) = 0; #endif CpvInitialize(int, cmiArgDebugFlag); CpvInitialize(char *, displayArgument); CpvInitialize(int, cpdSuspendStartup); CpvAccess(cmiArgDebugFlag) = 0; CpvAccess(displayArgument) = NULL; CpvAccess(cpdSuspendStartup) = 0; CcsBuiltinsInit(argv); rep_fw_handler_idx = CmiRegisterHandler((CmiHandler)rep_fw_handler); #if NODE_0_IS_CONVHOST #if ! CMK_CMIPRINTF_IS_A_BUILTIN print_fw_handler_idx = CmiRegisterHandler((CmiHandler)print_fw_handler); #endif { int ccs_serverPort=0; char *ccs_serverAuth=NULL; if (CmiGetArgFlagDesc(argv,"++server", "Create a CCS server port") | CmiGetArgIntDesc(argv,"++server-port",&ccs_serverPort, "Listen on this TCP/IP port number") | CmiGetArgStringDesc(argv,"++server-auth",&ccs_serverAuth, "Use this CCS authentication file")) if (CmiMyPe()==0) {/*Create and occasionally poll on a CCS server port*/ CcsServer_new(NULL,&ccs_serverPort,ccs_serverAuth); CcdCallOnConditionKeep(CcdPERIODIC,(CcdVoidFn)CcsServerCheck,NULL); } } #endif /* if in parallel debug mode i.e ++cpd, freeze */ if (CmiGetArgFlagDesc(argv, "+cpd", "Used *only* in conjunction with parallel debugger")) { CpvAccess(cmiArgDebugFlag) = 1; if (CmiGetArgStringDesc(argv, "+DebugDisplay",&(CpvAccess(displayArgument)), "X display for gdb used only in cpd mode")) { if (CpvAccess(displayArgument) == NULL) CmiPrintf("WARNING> NULL parameter for +DebugDisplay\n***"); } else if (CmiMyPe() == 0) { /* only one processor prints the warning */ CmiPrintf("WARNING> x term for gdb needs to be specified as +DebugDisplay by debugger\n***\n"); } if (CmiGetArgFlagDesc(argv, "+DebugSuspend", "Suspend execution at beginning of program")) { CpvAccess(cpdSuspendStartup) = 1; } } CcsReleaseMessages(); }
CcsHandlerRec *CcsGetHandler(const char *name) { return CkHashtableGet(CpvAccess(ccsTab),(void *)&name); }
void CcsRegisterHandler(const char *name, CmiHandler fn) { CcsHandlerRec cp; initHandlerRec(&cp,name); cp.fnOld=fn; *(CcsHandlerRec *)CkHashtablePut(CpvAccess(ccsTab),(void *)&cp.name)=cp; }
void CcsSendReplyNoError(int replyLen, const void *replyData) { if (CpvAccess(ccsReq)==NULL) return; CcsSendReply(replyLen, replyData); }
void CcsCallerId(skt_ip_t *pip, unsigned int *pport) { *pip = CpvAccess(ccsReq)->attr.ip; *pport = ChMessageInt(CpvAccess(ccsReq)->attr.port); }
//! process command line arguments! void TraceCounter::traceInit(char **argv) { CpvInitialize(CountLogPool*, _logPool); CpvInitialize(char*, _logName); CpvInitialize(double, version); CpvInitialize(char**, _counterNames); CpvInitialize(char**, _counterDesc); CpvInitialize(int, _numCounters); CpvInitialize(int, _reductionID); CpvAccess(_logName) = (char *) malloc(strlen(argv[0])+1); _MEMCHECK(CpvAccess(_logName)); strcpy(CpvAccess(_logName), argv[0]); CpvAccess(version) = VER; int i; // parse command line args char* counters = NULL; commandLine_ = NULL; bool badArg = false; int numCounters = 0; if (CmiGetArgStringDesc(argv, "+counters", &counters, "Measure these performance counters")) { if (CmiMyPe()==0) { CmiPrintf("Counters: %s\n", counters); } int offset = 0; int limit = strlen(counters); char* ptr = counters; while (offset < limit && (ptr = strtok(&counters[offset], ",")) != NULL) { offset += strlen(ptr)+1; ptr = &ptr[strlen(ptr)+1]; numCounters++; } if (CmiMyPe()==0) { CmiPrintf("There are %d counters\n", numCounters); } commandLine_ = new CounterArg[numCounters]; ptr = counters; for (i=0; i<numCounters; i++) { commandLine_[i].arg = ptr; if (!matchArg(&commandLine_[i])) { if (CmiMyPe()==0) { CmiPrintf("Bad arg: [%s]\n", ptr); } badArg = true; } ptr = &ptr[strlen(ptr)+1]; } } commandLineSz_ = numCounters; // check to see if args are valid, output if not if (badArg || CmiGetArgFlagDesc(argv, "+count-help", "List available performance counters")) { if (CmiMyPe() == 0) { printHelp(); } ConverseExit(); return; } else if (counters == NULL) { if (CmiMyPe() == 0) { usage(); } ConverseExit(); return; } // get optional command line args overview_ = CmiGetArgFlag(argv, "+count-overview"); switchRandom_ = CmiGetArgFlag(argv, "+count-switchrandom"); switchByPhase_ = CmiGetArgFlag(argv, "+count-switchbyphase"); noLog_ = CmiGetArgFlag(argv, "+count-nolog"); writeByPhase_ = CmiGetArgFlag(argv, "+count-writebyphase"); char* logName = NULL; if (CmiGetArgString(argv, "+count-logname", &logName)) { CpvAccess(_logName) = logName; if (noLog_) { if (CkMyPe()==0) { CmiPrintf("+count-logname and +count-nolog are MUTUALLY EXCLUSIVE\n"); usage(); CmiAbort(""); } } } if (switchByPhase_ && overview_) { if (CkMyPe()==0) { CmiPrintf( "+count-switchbyphase and +count-overview are MUTUALLY EXCLUSIVE\n" "+count-overview automatically switches by phase.\n"); usage(); CmiAbort(""); } } if (writeByPhase_ && noLog_) { if (CkMyPe()==0) { CmiPrintf("+count-writebyphase and +count-nolog are MUTUALLY EXCLUSIVE\n"); usage(); CmiAbort(""); } } // parse through commandLine_, figure out which belongs on which list (1 vs 2) CounterArg* last1 = NULL; CounterArg* last2 = NULL; CounterArg* tmp = NULL; counter1Sz_ = counter2Sz_ = 0; for (i=0; i<commandLineSz_; i++) { tmp = &commandLine_[i]; if (tmp->code < NUM_COUNTER_ARGS/2) { if (counter1_ == NULL) { counter1_ = tmp; last1 = counter1_; } else { last1->next = tmp; last1 = tmp; } counter1Sz_++; } else { if (counter2_ == NULL) { counter2_ = tmp; last2 = counter2_; } else { last2->next = tmp; last2 = tmp; } counter2Sz_++; } } if (counter1_ == NULL) { printHelp(); if (CmiMyPe()==0) { CmiPrintf("\nMust specify some counters with code < %d\n", NUM_COUNTER_ARGS/2); } ConverseExit(); } if (counter2_ == NULL) { printHelp(); if (CmiMyPe()==0) { CmiPrintf("\nMust specify some counters with code >= %d\n", NUM_COUNTER_ARGS/2); } ConverseExit(); } last1->next = counter1_; last2->next = counter2_; // all args valid, now set up logging if (CmiMyPe() == 0) { CmiPrintf("Running with tracemode=counter and args:\n"); // print out counter1 set tmp = counter1_; i = 0; do { CmiPrintf(" <counter1-%d>=%d %s %s\n", i, tmp->code, tmp->arg, tmp->desc); tmp = tmp->next; i++; } while (tmp != counter1_); // print out counter2 set tmp = counter2_; i = 0; do { CmiPrintf(" <counter2-%d>=%d %s %s\n", i, tmp->code, tmp->arg, tmp->desc); tmp = tmp->next; i++; } while (tmp != counter2_); CmiPrintf( "+count-overview %d\n+count-switchrandom %d\n" "+count-switchbyphase %d\n+count-nolog %d\n" "+count-logname %s\n+count-writebyphase %d\n", overview_, switchRandom_, switchByPhase_, noLog_, logName, writeByPhase_); } // DEBUGF((" DEBUG: Counter1=%d Counter2=%d\n", counter1_, counter2_)); CpvAccess(_logPool) = new CountLogPool(); // allocate names so can do reduction/analysis on the fly char** counterNames = new char*[counter1Sz_+counter2Sz_]; char** counterDesc = new char*[counter1Sz_+counter2Sz_]; tmp = counter1_; for (i=0; i<counter1Sz_; i++) { tmp->index = i; counterNames[i] = tmp->arg; counterDesc[i] = tmp->desc; tmp = tmp->next; } tmp = counter2_; for (i=0; i<counter2Sz_; i++) { tmp->index = counter1Sz_+i; counterNames[counter1Sz_+i] = tmp->arg; counterDesc[counter1Sz_+i] = tmp->desc; tmp = tmp->next; } CpvAccess(_counterNames) = counterNames; CpvAccess(_counterDesc) = counterDesc; CpvAccess(_numCounters) = numCounters; // don't erase counterNames or counterDesc, // the reduction client will do it on the final reduction _MEMCHECK(CpvAccess(_logPool)); CpvAccess(_logPool)->init(numCounters); DEBUGF(("%d/%d DEBUG: Created _logPool at %08x\n", CmiMyPe(), CmiNumPes(), CpvAccess(_logPool))); }
/** This is the main charm setup routine. It's called on all processors after Converse initialization. This routine gets passed to Converse from "main.C". The main purpose of this routine is to set up the objects and Ckpv's used during a regular Charm run. See the comment at the top of the file for overall flow. */ void _initCharm(int unused_argc, char **argv) { int inCommThread = (CmiMyRank() == CmiMyNodeSize()); DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer())); CkpvInitialize(size_t *, _offsets); CkpvAccess(_offsets) = new size_t[32]; CkpvInitialize(PtrQ*,_buffQ); CkpvInitialize(PtrVec*,_bocInitVec); CkpvInitialize(void*, _currentChare); CkpvInitialize(int, _currentChareType); CkpvInitialize(CkGroupID, _currentGroup); CkpvInitialize(void *, _currentNodeGroupObj); CkpvInitialize(CkGroupID, _currentGroupRednMgr); CkpvInitialize(GroupTable*, _groupTable); CkpvInitialize(GroupIDTable*, _groupIDTable); CkpvInitialize(CmiImmediateLockType, _groupTableImmLock); CkpvInitialize(bool, _destroyingNodeGroup); CkpvAccess(_destroyingNodeGroup) = false; CkpvInitialize(UInt, _numGroups); CkpvInitialize(int, _numInitsRecd); CkpvInitialize(int, _initdone); CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv; CkpvInitialize(MsgPool*, _msgPool); CkpvInitialize(CkCoreState *, _coreState); /* Added for evacuation-sayantan */ #ifndef __BIGSIM__ CpvInitialize(char *,_validProcessors); #endif CkpvInitialize(char ,startedEvac); CpvInitialize(int,serializer); _initChareTables(); // for checkpointable plain chares CksvInitialize(UInt, _numNodeGroups); CksvInitialize(GroupTable*, _nodeGroupTable); CksvInitialize(GroupIDTable, _nodeGroupIDTable); CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock); CksvInitialize(CmiNodeLock, _nodeLock); CksvInitialize(PtrVec*,_nodeBocInitVec); CksvInitialize(UInt,_numInitNodeMsgs); CkpvInitialize(int,_charmEpoch); CkpvAccess(_charmEpoch)=0; CksvInitialize(int, _triggersSent); CksvAccess(_triggersSent) = 0; CkpvInitialize(_CkOutStream*, _ckout); CkpvInitialize(_CkErrStream*, _ckerr); CkpvInitialize(Stats*, _myStats); CkpvAccess(_groupIDTable) = new GroupIDTable(0); CkpvAccess(_groupTable) = new GroupTable; CkpvAccess(_groupTable)->init(); CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock(); CkpvAccess(_numGroups) = 1; // make 0 an invalid group number CkpvAccess(_buffQ) = new PtrQ(); CkpvAccess(_bocInitVec) = new PtrVec(); CkpvAccess(_currentNodeGroupObj) = NULL; if(CkMyRank()==0) { CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number CksvAccess(_numInitNodeMsgs) = 0; CksvAccess(_nodeLock) = CmiCreateLock(); CksvAccess(_nodeGroupTable) = new GroupTable(); CksvAccess(_nodeGroupTable)->init(); CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock(); CksvAccess(_nodeBocInitVec) = new PtrVec(); } CkCallbackInit(); CmiNodeAllBarrier(); #if ! CMK_BIGSIM_CHARM initQd(argv); // bigsim calls it in ConverseCommonInit #endif CkpvAccess(_coreState)=new CkCoreState(); CkpvAccess(_numInitsRecd) = 0; CkpvAccess(_initdone) = 0; CkpvAccess(_ckout) = new _CkOutStream(); CkpvAccess(_ckerr) = new _CkErrStream(); _charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler); _initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler); CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState)); _roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler); _exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler); //added for interoperabilitY _libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler); _bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler); CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState)); #ifdef __BIGSIM__ if(BgNodeRank()==0) #endif _infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn); _triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler); _ckModuleInit(); CldRegisterEstimator((CldEstimator)_charmLoadEstimator); _futuresModuleInit(); // part of futures implementation is a converse module _loadbalancerInit(); _metabalancerInit(); #if CMK_MEM_CHECKPOINT init_memcheckpt(argv); #endif initCharmProjections(); #if CMK_TRACE_IN_CHARM // initialize trace module in ck traceCharmInit(argv); #endif CkpvInitialize(int, envelopeEventID); CkpvAccess(envelopeEventID) = 0; CkMessageWatcherInit(argv,CkpvAccess(_coreState)); /** The rank-0 processor of each node calls the translator-generated "_register" routines. _register routines call the charm.h "CkRegister*" routines, which record function pointers and class information for all Charm entities, like Chares, Arrays, and readonlies. There's one _register routine generated for each .ci file. _register routines *must* be called in the same order on every node, and *must not* be called by multiple threads simultaniously. */ #ifdef __BIGSIM__ if(BgNodeRank()==0) #else if(CkMyRank()==0) #endif { SDAG::registerPUPables(); CmiArgGroup("Charm++",NULL); _parseCommandLineOpts(argv); _registerInit(); CkRegisterMsg("System", 0, 0, CkFreeMsg, sizeof(int)); CkRegisterChareInCharm(CkRegisterChare("null", 0, TypeChare)); CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare), TypeChare); CkRegisterChareInCharm(CkIndex_Chare::__idx); CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group), TypeGroup); CkRegisterChareInCharm(CkIndex_Group::__idx); CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC); /** These _register calls are for the built-in Charm .ci files, like arrays and load balancing. If you add a .ci file to charm, you'll have to add a call to the _register routine here, or make your library into a "-module". */ _registerCkFutures(); _registerCkArray(); _registerLBDatabase(); _registerMetaBalancer(); _registerCkCallback(); _registertempo(); _registerwaitqd(); _registerCkCheckpoint(); #if CMK_MEM_CHECKPOINT _registerCkMemCheckpoint(); #endif /* Setup Control Point Automatic Tuning Framework. By default it is enabled as a part of charm, however it won't enable its tracing module unless a +CPEnableMeasurements command line argument is specified. See trace-common.C for more info Thus there should be no noticable overhead to always having the control point framework linked in. */ #if CMK_WITH_CONTROLPOINT _registerPathHistory(); _registerControlPoints(); _registerTraceControlPoints(); #endif /** CkRegisterMainModule is generated by the (unique) "mainmodule" .ci file. It will include calls to register all the .ci files. */ CkRegisterMainModule(); /** _registerExternalModules is actually generated by charmc at link time (as "moduleinit<pid>.C"). This generated routine calls the _register functions for the .ci files of libraries linked using "-module". This funny initialization is most useful for AMPI/FEM programs, which don't have a .ci file and hence have no other way to control the _register process. */ _registerExternalModules(argv); _registerDone(); } /* The following will happen on every virtual processor in BigEmulator, not just on once per real processor */ if (CkMyRank() == 0) { CpdBreakPointInit(); } CmiNodeAllBarrier(); // Execute the initcalls registered in modules _initCallTable.enumerateInitCalls(); #if CMK_CHARMDEBUG CpdFinishInitialization(); #endif //CmiNodeAllBarrier(); CkpvAccess(_myStats) = new Stats(); CkpvAccess(_msgPool) = new MsgPool(); CmiNodeAllBarrier(); #if !(__FAULT__) CmiBarrier(); CmiBarrier(); CmiBarrier(); #endif #if CMK_SMP_TRACE_COMMTHREAD _TRACE_BEGIN_COMPUTATION(); #else if (!inCommThread) { _TRACE_BEGIN_COMPUTATION(); } #endif #ifdef ADAPT_SCHED_MEM if(CkMyRank()==0){ memCriticalEntries = new int[numMemCriticalEntries]; int memcnt=0; for(int i=0; i<_entryTable.size(); i++){ if(_entryTable[i]->isMemCritical){ memCriticalEntries[memcnt++] = i; } } } #endif #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) _messageLoggingInit(); #endif #ifndef __BIGSIM__ /* FAULT_EVAC */ CpvAccess(_validProcessors) = new char[CkNumPes()]; for(int vProc=0;vProc<CkNumPes();vProc++){ CpvAccess(_validProcessors)[vProc]=1; } _ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast); _ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac); #endif CkpvAccess(startedEvac) = 0; CpvAccess(serializer) = 0; evacuate = 0; CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0); #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) CcdCallOnCondition(CcdSIGUSR2,(CcdVoidFn)CkMlogRestart,0); #endif if(_raiseEvac){ processRaiseEvacFile(_raiseEvacFile); /* if(CkMyPe() == 2){ // CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0); CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000); } if(CkMyPe() == 3){ CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000); }*/ } if (CkMyRank() == 0) { TopoManager_init(); } CmiNodeAllBarrier(); if (!_replaySystem) { CkFtFn faultFunc_restart = CkRestartMain; if (faultFunc == NULL || faultFunc == faultFunc_restart) { // this is not restart from memory // these two are blocking calls for non-bigsim #if ! CMK_BIGSIM_CHARM CmiInitCPUAffinity(argv); CmiInitMemAffinity(argv); #endif } CmiInitCPUTopology(argv); #if CMK_SHARED_VARS_POSIX_THREADS_SMP if (CmiCpuTopologyEnabled()) { int *pelist; int num; CmiGetPesOnPhysicalNode(0, &pelist, &num); #if !CMK_MULTICORE && !CMK_SMP_NO_COMMTHD // Count communication threads, if present // XXX: Assuming uniformity of node size here num += num/CmiMyNodeSize(); #endif if (!_Cmi_forceSpinOnIdle && num > CmiNumCores()) { if (CmiMyPe() == 0) CmiPrintf("\nCharm++> Warning: the number of SMP threads (%d) is greater than the number of physical cores (%d), so threads will sleep while idling. Use +CmiSpinOnIdle or +CmiSleepOnIdle to control this directly.\n\n", num, CmiNumCores()); CmiLock(CksvAccess(_nodeLock)); if (! _Cmi_sleepOnIdle) _Cmi_sleepOnIdle = 1; CmiUnlock(CksvAccess(_nodeLock)); } } #endif } if(CmiMyPe() == 0) { char *topoFilename; if(CmiGetArgStringDesc(argv,"+printTopo",&topoFilename,"topo file name")) { std::stringstream sstm; sstm << topoFilename << "." << CmiMyPartition(); std::string result = sstm.str(); FILE *fp; fp = fopen(result.c_str(), "w"); if (fp == NULL) { CkPrintf("Error opening %s file, writing to stdout\n", topoFilename); fp = stdout; } TopoManager_printAllocation(fp); fclose(fp); } } #if CMK_USE_PXSHM && ( CMK_CRAYXE || CMK_CRAYXC ) && CMK_SMP // for SMP on Cray XE6 (hopper) it seems pxshm has to be initialized // again after cpuaffinity is done if (CkMyRank() == 0) { CmiInitPxshm(argv); } CmiNodeAllBarrier(); #endif //CldCallback(); #if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG // Register the BG handler for CCS. Notice that this is put into a variable shared by // the whole real processor. This because converse needs to find it. We check that all // virtual processors register the same index for this handler. CpdBgInit(); #endif if (faultFunc) { #if CMK_WITH_STATS if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()]; #endif if (!inCommThread) { CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0); msg->argc = CmiGetArgc(argv); msg->argv = argv; faultFunc(_restartDir, msg); CkFreeMsg(msg); } }else if(CkMyPe()==0){ #if CMK_WITH_STATS _allStats = new Stats*[CkNumPes()]; #endif register size_t i, nMains=_mainTable.size(); for(i=0;i<nMains;i++) /* Create all mainchares */ { register int size = _chareTable[_mainTable[i]->chareIdx]->size; register void *obj = malloc(size); _MEMCHECK(obj); _mainTable[i]->setObj(obj); CkpvAccess(_currentChare) = obj; CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx; register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0); msg->argc = CmiGetArgc(argv); msg->argv = argv; _entryTable[_mainTable[i]->entryIdx]->call(msg, obj); #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) CpvAccess(_currentObj) = (Chare *)obj; #endif } _mainDone = 1; _STATS_RECORD_CREATE_CHARE_N(nMains); _STATS_RECORD_PROCESS_CHARE_N(nMains); for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */ { register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg)); if(roMsg==0) continue; //Pack the message and send it to all other processors register envelope *env = UsrToEnv(roMsg); env->setSrcPe(CkMyPe()); env->setMsgtype(ROMsgMsg); env->setRoIdx(i); CmiSetHandler(env, _initHandlerIdx); CkPackMessage(&env); CmiSyncBroadcast(env->getTotalsize(), (char *)env); CpvAccess(_qd)->create(CkNumPes()-1); //For processor 0, unpack and re-set the global CkUnpackMessage(&env); _processROMsgMsg(env); _numInitMsgs++; } //Determine the size of the RODataMessage PUP::sizer ps; for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps); //Allocate and fill out the RODataMessage envelope *env = _allocEnv(RODataMsg, ps.size()); PUP::toMem pp((char *)EnvToUsr(env)); for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp); env->setCount(++_numInitMsgs); env->setSrcPe(CkMyPe()); CmiSetHandler(env, _initHandlerIdx); DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize())); CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env); CpvAccess(_qd)->create(CkNumPes()-1); _initDone(); } DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread)); // when I am a communication thread, I don't participate initDone. if (inCommThread) { CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler, CkpvAccess(_coreState)); CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler , CkpvAccess(_coreState)); _processBufferedMsgs(); } #if CMK_CHARMDEBUG // Should not use CpdFreeze inside a thread (since this processor is really a user-level thread) if (CpvAccess(cpdSuspendStartup)) { //CmiPrintf("In Parallel Debugging mode .....\n"); CpdFreeze(); } #endif #if __FAULT__ if(killFlag){ readKillFile(); } #endif }
//! end computation, do a reduction here in hopes that it finishes before //! traceClose called and the program exits void TraceCounter::endComputation() { CpvAccess(_logPool)->doReduction(-1, idleTime_); }
/// Basic Constructor PVT::PVT() { #ifdef VERBOSE_DEBUG CkPrintf("[%d] constructing PVT\n",CkMyPe()); #endif CpvInitialize(int, stateRecovery); CpvAccess(stateRecovery) = 0; CpvInitialize(eventID, theEventID); CpvAccess(theEventID)=eventID(); // CpvAccess(theEventID).dump(); //LBTurnInstrumentOff(); optGVT = POSE_UnsetTS; conGVT = POSE_UnsetTS; rdone=0; SRs=NULL; #ifdef POSE_COMM_ON //com_debug = 1; #endif #ifndef CMK_OPTIMIZE localStats = (localStat *)CkLocalBranch(theLocalStats); if (pose_config.stats) { localStats->TimerStart(GVT_TIMER); } #endif #ifdef MEM_TEMPORAL localTimePool = (TimePool *)CkLocalBranch(TempMemID); CkPrintf("NOTE: Temporal memory manager is ON!\n"); #endif optPVT = conPVT = estGVT = POSE_UnsetTS; startPhaseActive = gvtTurn = simdone = 0; SendsAndRecvs = new SRtable(); SendsAndRecvs->Initialize(); specEventCount = eventCount = waitForFirst = 0; iterMin = POSE_UnsetTS; int P=CkNumPes(), N=CkMyPe(); reportReduceTo = -1; if ((N < P-2) && (N%2 == 1)) { //odd reportTo = N-1; reportsExpected = reportEnd = 0; } else if (N < P-2) { //even reportTo = N; reportsExpected = 2; if (N == P-3) reportsExpected = 1; reportEnd = 0; if (N < (P-2)/2) reportReduceTo = P-2; else reportReduceTo = P-1; } if (N == P-2) { reportTo = N; reportEnd = 1; reportsExpected = 1 + (P-2)/4 + ((P-2)%4)/2; } else if (N == P-1) { reportTo = N; reportEnd = 1; if (P==1) reportsExpected = 1; else reportsExpected = 1 + (P-2)/4 + (P-2)%2; } // CkPrintf("PE %d reports to %d, receives %d reports, reduces and sends to %d, and reports directly to GVT if %d = 1!\n", CkMyPe(), reportTo, reportsExpected, reportReduceTo, reportEnd); parCheckpointInProgress = 0; parLastCheckpointGVT = 0; parLastCheckpointTime = parStartTime = 0.0; parLBInProgress = 0; parLastLBGVT = 0; // debugBufferLoc = debugBufferWrapped = debugBufferDumped = 0; #ifndef CMK_OPTIMIZE if(pose_config.stats) localStats->TimerStop(); #endif LBDatabase::Object()->AddMigrationDoneFn(staticDoneLB, this); }
//! write the summary sts file for this trace void TraceCounter::traceWriteSts() { if (traceOn_ && !noLog_) { if (CmiMyPe()==0) { CpvAccess(_logPool)->writeSts(); } } }
void resumeCentralLbAfterChkpt(void *_lb){ CentralLB *lb= (CentralLB *)_lb; CpvAccess(_currentObj)=lb; lb->endMigrationDone(lb->savedBalancing); }
int pvmc_unpackmsg(void *msgbuf, void *start_of_msg) { pvmc_buffer *cur_buf; pvmc_item *cur_item, *nxt_item; int bytes_unpacked=0; int i; if ((CpvAccess(pvmc_rbufid)<=0) || (CpvAccess(pvmc_rbufid) >= MAX_BUFFERS) || (CpvAccess(pvmc_bufarray)[CpvAccess(pvmc_rbufid)].refcount <= 0)) { PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() uninitialized recv buffer\n", MYPE(),__FILE__,__LINE__); return -1; } cur_buf = &(CpvAccess(pvmc_bufarray)[CpvAccess(pvmc_rbufid)]); pvmc_emptybuf(cur_buf); cur_buf->bytes = *((int *)((char *)start_of_msg+bytes_unpacked)); bytes_unpacked += sizeof(int); cur_buf->tag = *((int *)((char *)start_of_msg+bytes_unpacked)); bytes_unpacked += sizeof(int); cur_buf->tid = *((int *)((char *)start_of_msg+bytes_unpacked)); bytes_unpacked += sizeof(int); cur_buf->num_items = *((int *)((char *)start_of_msg+bytes_unpacked)); bytes_unpacked += sizeof(int); #ifdef PVM_DEBUG PRINTF("Pe(%d) pvmc_unpackmsg: %d items unpacked for tag %d\n", MYPE(),cur_buf->num_items,cur_buf->tag); #endif if (msgbuf) cur_buf->data_buf = msgbuf; else cur_buf->data_buf = (void *)NULL; cur_item=(pvmc_item *)MALLOC(sizeof(pvmc_item)); cur_buf->first_item=cur_item; cur_buf->cur_item=cur_item; if (cur_item==(pvmc_item *)NULL) { PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() can't allocate memory\n", MYPE(),pvm_mytid(),__FILE__,__LINE__); return -1; } #if PVM_DEBUG PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() unpacking %d messages.\n", MYPE(),pvm_mytid(),__FILE__,__LINE__,cur_buf->num_items); #endif for(i=0;i<cur_buf->num_items;i++) { cur_item->type = *((int *)((char *)start_of_msg+bytes_unpacked)); bytes_unpacked+=sizeof(int); cur_item->size = *((int *)((char *)start_of_msg+bytes_unpacked)); bytes_unpacked+=sizeof(int); nxt_item=(pvmc_item *)MALLOC(sizeof(pvmc_item)); if (!nxt_item) { PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() can't allocate memory\n", MYPE(),pvm_mytid(),__FILE__,__LINE__); return -1; } cur_item->nxt = nxt_item; cur_item = nxt_item; } cur_item->type = 0; cur_item->size = 0; cur_item->free_data = FALSE; cur_item->data = (char *) NULL; cur_item->nxt = (pvmc_item *) NULL; cur_buf->last_item = cur_item; cur_item = cur_buf->first_item; while(cur_item!=cur_buf->last_item) { if (cur_item->size > 0) { cur_item->free_data=FALSE; cur_item->data = (void *)((char *)start_of_msg+bytes_unpacked); bytes_unpacked+=cur_item->size; } else cur_item->data=NULL; cur_item = cur_item->nxt; } return bytes_unpacked; }
void CpdUnFreeze(void) { CpvAccess(freezeModeFlag) = 0; }
CmiStartFn mymain(int argc, char** argv) { if(CmiMyRank() == CmiMyNodeSize()) return 0; CpvInitialize(int,msgSize); CpvInitialize(int,cycleNum); CpvInitialize(int,sizeNum); CpvAccess(sizeNum) = 1; CpvAccess(msgSize)= CmiMsgHeaderSizeBytes + 8; CpvInitialize(int,exitHandler); CpvAccess(exitHandler) = CmiRegisterHandler((CmiHandler) exitHandlerFunc); CpvInitialize(int,node0Handler); CpvAccess(node0Handler) = CmiRegisterHandler((CmiHandler) node0HandlerFunc); CpvInitialize(int,node1Handler); CpvAccess(node1Handler) = CmiRegisterHandler((CmiHandler) node1HandlerFunc); CpvInitialize(int,ackHandler); CpvAccess(ackHandler) = CmiRegisterHandler((CmiHandler) ackHandlerFunc); CpvInitialize(double,startTime); CpvInitialize(double,endTime); CpvInitialize(double, IdleStartTime); CpvInitialize(double, IdleTime); CpvInitialize(int,ackCount); CpvAccess(ackCount) = 0; CpvInitialize(int,twoway); CpvAccess(twoway) = 0; CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE, ApplIdleStart, NULL); CcdCallOnConditionKeep(CcdPROCESSOR_END_IDLE, ApplIdleEnd, NULL); if(argc > 1) CpvAccess(twoway) = atoi(argv[1]); if(CmiMyPe() == 0) { if(!CpvAccess(twoway)) CmiPrintf("Starting Pingpong with oneway traffic \n"); else CmiPrintf("Starting Pingpong with twoway traffic\n"); } if ((CmiMyPe() < CmiNumPes()/2) || CpvAccess(twoway)) startPingpong(); return 0; }
int CpdIsFrozen(void) { return CpvAccess(freezeModeFlag); }
/** * Register a callback function that will be triggered *whenever* the specified * condition is raised */ int CcdCallOnConditionKeep(int condnum, CcdVoidFn fnp, void *arg) { return append_elem(&(CpvAccess(conds).condcb_keep[condnum]), fnp, arg, CcdIGNOREPE); }
//! do a reduction across processors to calculate the total count for //! each count, and if the count has flops, etc, then calc the //! the flops/s, etc... void StatTable::doReduction(int phase, double idleTime) { DEBUGF(("%d/%d DEBUG: StatTable::doReduction()\n", CmiMyPe(), CmiNumPes(), this)); // see above (NUM_EXTRA_PERF) for the fields in the message int msgSize = ALIGN8(CmiMsgHeaderSizeBytes)+ sizeof(double)*(2*numStats_+NUM_EXTRA_PERF); char *msg = (char *)CmiAlloc(msgSize); double* reduction = (double*)(msg+ALIGN8(CmiMsgHeaderSizeBytes)); // calculate flops/s, l1%, l2%, tlb% if it's there char** counterNames = CpvAccess(_counterNames); int GR_FLOPS = -1; double flopsRate = -1.0; int LOAD = -1; double loadRate = -1.0; int STORE = -1; double storeRate = -1.0; int L1_DMISS = -1; double l1Rate = -1.0; int L2_DMISS = -1; double l2Rate = -1.0; int TLB_MISS = -1; double tlbRate = -1.0; int i, j; for (i=0; i<2*numStats_+NUM_EXTRA_PERF; i++) { reduction[i] = 0.0; } for (i=0; i<numStats_; i++) { for (int j=0; j<MAX_ENTRIES; j++) { reduction[2*i] += stats_[i].numCalled[j]*stats_[i].avgCount[j]; reduction[2*i+1] += stats_[i].totTime[j]; } if (strcmp(counterNames[i], "GR_FLOPS")==0) { GR_FLOPS = i; } else if (strcmp(counterNames[i], "LOAD")==0) { LOAD = i; } else if (strcmp(counterNames[i], "STORE")==0) { STORE = i; } else if (strcmp(counterNames[i], "L1_DMISS")==0) { L1_DMISS = i; } else if (strcmp(counterNames[i], "L2_DMISS")==0) { L2_DMISS = i; } else if (strcmp(counterNames[i], "TLB_MISS")==0) { TLB_MISS = i; } } if (CmiMyPe()==0) { reduction[2*numStats_] = phase; } reduction[2*numStats_+1] = idleTime; // -1 for the rest of the calc values reduction[2*numStats_+2] = -1.0; reduction[2*numStats_+3] = -1.0; reduction[2*numStats_+4] = -1.0; reduction[2*numStats_+5] = -1.0; // calculate flops/s, l1%, l2%, tlb% if it's there double* rate = NULL; int index; for (i=0; i<6; i++) { switch (i) { case 0: rate = &flopsRate; index = GR_FLOPS; break; case 1: rate = &loadRate; index = LOAD; break; case 2: rate = &storeRate; index = STORE; break; case 3: rate = &l1Rate; index = L1_DMISS; break; case 4: rate = &l2Rate; index = L2_DMISS; break; case 5: rate = &tlbRate; index = TLB_MISS; break; } if (index >= 0 && reduction[2*index+1] > 0.0) { // if we have the counter AND it's times were non-zero *rate = reduction[2*index]/reduction[2*index+1]; } } // store rates if there if (GR_FLOPS >= 0) { reduction[2*numStats_+2] = flopsRate; } if (LOAD >= 0 && STORE >= 0) { double memRate = loadRate + storeRate; if (L1_DMISS >= 0 & memRate > 0) { reduction[2*numStats_+3] = l1Rate / memRate; } if (L2_DMISS >= 0 & memRate > 0) { reduction[2*numStats_+4] = l2Rate / memRate; } if (TLB_MISS >= 0 & memRate > 0) { reduction[2*numStats_+5] = tlbRate / memRate; } } // send the data CmiSetHandler(msg, (int)CpvAccess(_reductionID)); int handlerID = CmiGetHandler(msg); DEBUGF(("%d/%d handlerID %d reductionID %d\n", CmiMyPe(), CmiNumPes(), handlerID, CpvAccess(_reductionID))); CmiSyncSendAndFree(0, msgSize, msg); }
/** * Cancel a previously registered conditional callback */ void CcdCancelCallOnConditionKeep(int condnum, int idx) { remove_elem(&(CpvAccess(conds).condcb_keep[condnum]), idx); }
// a rudimentary reduction to print out the performance results across the run CmiHandler StatTableReduction(char* msg) { DEBUGF(("StatTableReduction called\n", CmiMyPe(), CmiNumPes())); static double* reduce = NULL; static int numReduce = 0; int numCounters = CpvAccess(_numCounters); int size = 2*CpvAccess(_numCounters)+NUM_EXTRA_PERF; int i; if (reduce == NULL) { // allocate reduce = new double[size]; for (i=0; i<size; i++) { reduce[i] = 0.0; } DEBUGF((" allocated reduce numCounters %d size %d\n", numCounters, size)); } // see above for the feilds of this message double* msgResults = (double *)(msg+ALIGN8(CmiMsgHeaderSizeBytes)); for (i=0; i<size; i++) { reduce[i] += msgResults[i]; } char** counterNames = CpvAccess(_counterNames); numReduce++; DEBUGF((" numReduce %d numPes %d\n", numReduce, CmiNumPes())); int phase = reduce[2*numCounters]; if (numReduce >= CmiNumPes()) { // finished with reduction, print out results numReduce = 0; for (i=0; i<numCounters; i++) { if (reduce[2*i+1]>0.0) { // is time > 0? if (phase >= 0) { CmiPrintf("PHASE %d %s totalCount %f totalTime (us) %f\n" "PHASE %d %s count/proc %f avgTime (us)/phase %f\n", phase, counterNames[i], reduce[2*i], reduce[2*i+1]*1e6, phase, counterNames[i], reduce[2*i]/CmiNumPes(), reduce[2*i+1]*1e6/CmiNumPes()); } else { CmiPrintf("%s totalCount %f totalTime (us) %f\n" "%s count/proc %f avgTime (us)/phase %f\n", counterNames[i], reduce[2*i], reduce[2*i+1]*1e6, counterNames[i], reduce[2*i]/CmiNumPes(), reduce[2*i+1]*1e6/CmiNumPes()); } } } if (phase >= 0) { CmiPrintf("PHASE %d totalIdleTime (us) %f avgIdleTime (us)/phase %f\n", phase, reduce[2*numCounters+1]*1e6, reduce[2*numCounters+1]*1e6/CmiNumPes()); } else { CmiPrintf("totalIdleTime (us) %f avgIdleTime (us)/phase %f\n", reduce[2*numCounters+1]*1e6, reduce[2*numCounters+1]*1e6/CmiNumPes()); } if (reduce[2*numCounters+2] > 0.0) { // we have flops if (phase >= 0) { CmiPrintf("PHASE %d flops/s %f flops/s/PE %f\n", phase, reduce[2*numCounters+2], reduce[2*numCounters+2]/CmiNumPes()); } else { CmiPrintf("flops/s %f flops/s/PE %f\n", reduce[2*numCounters+2], reduce[2*numCounters+2]/CmiNumPes()); } } char* missRate = NULL; for (i=0; i<3; i++) { switch (i) { case 0: missRate = "l1 avg miss rate (%)"; break; case 1: missRate = "l2 avg miss rate (%)"; break; case 2: missRate = "tlb avg miss rate (%)"; break; } if (reduce[2*numCounters+3+i] >= 0.0) { if (phase >= 0) { CmiPrintf("PHASE %d %s %f\n", phase, missRate, reduce[2*numCounters+3+i]/CmiNumPes()*100); } else { CmiPrintf("%s %f\n", missRate, reduce[2*numCounters+3+i]/CmiNumPes()*100); } } } // clean up delete [] reduce; reduce = NULL; } CmiFree(msg); }
/** * Called when something drastic changes-- restart ccd_num_checks */ void CcdCallBacksReset(void *ignored,double curWallTime) { ccd_periodic_callbacks *o=&CpvAccess(pcb); CpvAccess(_ccd_numchecks)=o->nSkip=1; o->lastCheck=curWallTime; }
void CtgInit(void) { CpvInitialize(int, CmiPICMethod); CpvAccess(CmiPICMethod) = CMI_PIC_NOP; }