int CcsIsRemoteRequest(void)
{
return CpvAccess(ccsReq)!=NULL;
}
/**
* Register a callback function that will be triggered on the specified PE
* *whenever* the specified condition is raised
*/
int CcdCallOnConditionKeepOnPE(int condnum, CcdVoidFn fnp, void *arg, int pe)
{
return append_elem(&(CpvAccess(conds).condcb_keep[condnum]), fnp, arg, pe);
}
/**
* Raise a condition causing all registered callbacks corresponding to
* that condition to be triggered
*/
void CcdRaiseCondition(int condnum)
{
double curWallTime=CmiWallTimer();
call_cblist_remove(&(CpvAccess(conds).condcb[condnum]),curWallTime);
call_cblist_keep(&(CpvAccess(conds).condcb_keep[condnum]),curWallTime);
}
static void collectNumbers(ProcMsg *msg)
{
int npes;
EmptyMsg emsg;
if(CpvAccess(isSingle)) {
CpvAccess(Time1) = CmiWallTimer() - CpvAccess(Time1);
CpvAccess(seqPI) = 4.0 * msg->success / NTRIALS;
CpvAccess(isSingle) = 0;
CpvAccess(nreported) = 0;
CpvAccess(success) = 0;
msg->success = NTRIALS/CmiNumPes();
CmiSetHandler(msg, CpvAccess(trial_handler));
CmiSyncBroadcastAll(sizeof(ProcMsg), msg);
CpvAccess(TimeN) = CmiWallTimer();
printf("if\n");
} else {
printf("else\n");
CpvAccess(nreported)++;
CpvAccess(success) += msg->success;
if(CpvAccess(nreported)==CmiNumPes()) {
CpvAccess(TimeN) = CmiWallTimer() - CpvAccess(TimeN);
CpvAccess(parPI) = 4.0 * CpvAccess(success) / NTRIALS;
npes = iround(CpvAccess(Time1)/CpvAccess(TimeN));
CmiPrintf("[proc] Tseq = %le seconds, Tpar = %le seconds\n",
CpvAccess(Time1), CpvAccess(TimeN));
CmiPrintf("[proc] CmiNumPes() reported %d processors\n", CmiNumPes());
CmiPrintf("[proc] But actual number of processors is %d\n", npes);
CmiPrintf("[proc] FYI, appox PI (seq) = %lf\n",CpvAccess(seqPI));
CmiPrintf("[proc] FYI, appox PI (par) = %lf\n",CpvAccess(parPI));
CmiSetHandler(&emsg, CpvAccess(ack_handler));
CmiSyncSend(0, sizeof(EmptyMsg), &emsg);
printf("else if\n");
}
}
}
//Registering idle handlers
void ApplIdleStart(void *, double start)
{
CpvAccess(IdleStartTime)= start; //CmiWallTimer();
return;
}
//! clear all data collected for entry points
void TraceCounter::traceClearEps() {
CpvAccess(_logPool)->clearEps();
}
void pvmc_init_bufs(void)
{
int i;
#ifdef PVM_DEBUG
PRINTF("Pe(%d) tid=%d:%s:%d pvmc_init_bufs() initializing buffer array\n",
MYPE(),pvm_mytid(),__FILE__,__LINE__);
#endif
CpvInitialize(pvmc_buffer*,pvmc_bufarray);
CpvAccess(pvmc_bufarray)=MALLOC(sizeof(pvmc_buffer)*MAX_BUFFERS);
if (CpvAccess(pvmc_bufarray)==NULL) {
PRINTF("Pe(%d) tid=%d:%s:%d pvmc_init_bufs() can't alloc buffer array\n",
MYPE(),pvm_mytid(),__FILE__,__LINE__);
exit(1);
}
CpvInitialize(pvmc_buffer*,pvmc_freebufs);
CpvAccess(pvmc_freebufs)=&(CpvAccess(pvmc_bufarray)[1]); /* throw away first bufid */
for(i=0;i<MAX_BUFFERS;i++) {
CpvAccess(pvmc_bufarray)[i].bufid=i;
CpvAccess(pvmc_bufarray)[i].bytes=0;
CpvAccess(pvmc_bufarray)[i].tag=0;
CpvAccess(pvmc_bufarray)[i].tid=-1;
CpvAccess(pvmc_bufarray)[i].num_items=-1;
CpvAccess(pvmc_bufarray)[i].refcount=0;
CpvAccess(pvmc_bufarray)[i].first_item=(pvmc_item *)NULL;
CpvAccess(pvmc_bufarray)[i].cur_item=(pvmc_item *)NULL;
CpvAccess(pvmc_bufarray)[i].last_item=(pvmc_item *)NULL;
if (i==MAX_BUFFERS-1)
CpvAccess(pvmc_bufarray)[i].nxt_free=(pvmc_buffer *)NULL;
else
CpvAccess(pvmc_bufarray)[i].nxt_free=&(CpvAccess(pvmc_bufarray)[i+1]);
CpvAccess(pvmc_bufarray)[i].data_buf=(char *)NULL;
}
CpvInitialize(int,pvmc_sbufid);
CpvAccess(pvmc_sbufid) = 0;
CpvInitialize(int,pvmc_rbufid);
CpvAccess(pvmc_rbufid) = 0;
}
static inline void _parseCommandLineOpts(char **argv)
{
if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown"))
_STATS_ON(_printCS);
if (CmiGetArgFlagDesc(argv,"+ss", "Print summary statistics at shutdown"))
_STATS_ON(_printSS);
if (CmiGetArgFlagDesc(argv,"+fifo", "Default to FIFO queuing"))
_defaultQueueing = CK_QUEUEING_FIFO;
if (CmiGetArgFlagDesc(argv,"+lifo", "Default to LIFO queuing"))
_defaultQueueing = CK_QUEUEING_LIFO;
if (CmiGetArgFlagDesc(argv,"+ififo", "Default to integer-prioritized FIFO queuing"))
_defaultQueueing = CK_QUEUEING_IFIFO;
if (CmiGetArgFlagDesc(argv,"+ilifo", "Default to integer-prioritized LIFO queuing"))
_defaultQueueing = CK_QUEUEING_ILIFO;
if (CmiGetArgFlagDesc(argv,"+bfifo", "Default to bitvector-prioritized FIFO queuing"))
_defaultQueueing = CK_QUEUEING_BFIFO;
if (CmiGetArgFlagDesc(argv,"+blifo", "Default to bitvector-prioritized LIFO queuing"))
_defaultQueueing = CK_QUEUEING_BLIFO;
if (CmiGetArgFlagDesc(argv,"+objq", "Default to use object queue for every obejct"))
{
#if CMK_OBJECT_QUEUE_AVAILABLE
_defaultObjectQ = 1;
if (CkMyPe()==0)
CmiPrintf("Charm++> Create object queue for every Charm object.\n");
#else
CmiAbort("Charm++> Object queue not enabled, recompile Charm++ with CMK_OBJECT_QUEUE_AVAILABLE defined to 1.");
#endif
}
if(CmiGetArgString(argv,"+restart",&_restartDir))
faultFunc = CkRestartMain;
#if __FAULT__
if (CmiGetArgIntDesc(argv,"+restartaftercrash",&CpvAccess(_curRestartPhase),"restarting this processor after a crash")){
# if CMK_MEM_CHECKPOINT
faultFunc = CkMemRestart;
# endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
faultFunc = CkMlogRestart;
#endif
CmiPrintf("[%d] Restarting after crash \n",CmiMyPe());
}
#if CMK_MESSAGE_LOGGING
// reading +ftc_disk flag
if (CmiGetArgFlagDesc(argv, "+ftc_disk", "Disk Checkpointing")) {
diskCkptFlag = 1;
}
#endif
// reading the killFile
if(CmiGetArgStringDesc(argv,"+killFile", &killFile,"Generates SIGKILL on specified processors")){
if(faultFunc == NULL){
//do not read the killfile if this is a restarting processor
killFlag = 1;
if(CmiMyPe() == 0){
printf("[%d] killFlag set to 1 for file %s\n",CkMyPe(),killFile);
}
}
}
#endif
// shut down program in ring fashion to allow projections output w/o IO error
if (CmiGetArgIntDesc(argv,"+ringexit",&_ringtoken, "Program exits in a ring fashion"))
{
_ringexit = 1;
if (CkMyPe()==0)
CkPrintf("Charm++> Program shutdown in token ring (%d).\n", _ringtoken);
if (_ringtoken > CkNumPes()) _ringtoken = CkNumPes();
}
/*
FAULT_EVAC
if the argument +raiseevac is present then cause faults
*/
if(CmiGetArgStringDesc(argv,"+raiseevac", &_raiseEvacFile,"Generates processor evacuation on random processors")){
_raiseEvac = 1;
}
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
if(!CmiGetArgIntDesc(argv,"+teamSize",&teamSize,"Set the team size for message logging")){
teamSize = 1;
}
if(!CmiGetArgIntDesc(argv,"+chkptPeriod",&chkptPeriod,"Set the checkpoint period for the message logging fault tolerance algorithm in seconds")){
chkptPeriod = 100;
}
if(CmiGetArgIntDesc(argv,"+fastRecovery", ¶llelRecovery, "Parallel recovery with message logging protocol")){
fastRecovery = true;
}
#endif
/* Anytime migration flag */
_isAnytimeMigration = true;
if (CmiGetArgFlagDesc(argv,"+noAnytimeMigration","The program does not require support for anytime migration")) {
_isAnytimeMigration = false;
}
_isNotifyChildInRed = true;
if (CmiGetArgFlagDesc(argv,"+noNotifyChildInReduction","The program has at least one element per processor for each charm array created")) {
_isNotifyChildInRed = false;
}
_isStaticInsertion = false;
if (CmiGetArgFlagDesc(argv,"+staticInsertion","Array elements are only inserted at construction")) {
_isStaticInsertion = true;
}
useNodeBlkMapping = false;
if (CmiGetArgFlagDesc(argv,"+useNodeBlkMapping","Array elements are block-mapped in SMP-node level")) {
useNodeBlkMapping = true;
}
#if ! CMK_WITH_CONTROLPOINT
// Display a warning if charm++ wasn't compiled with control point support but user is expecting it
if( CmiGetArgFlag(argv,"+CPSamplePeriod") ||
CmiGetArgFlag(argv,"+CPSamplePeriodMs") ||
CmiGetArgFlag(argv,"+CPSchemeRandom") ||
CmiGetArgFlag(argv,"+CPExhaustiveSearch") ||
CmiGetArgFlag(argv,"+CPAlwaysUseDefaults") ||
CmiGetArgFlag(argv,"+CPSimulAnneal") ||
CmiGetArgFlag(argv,"+CPCriticalPathPrio") ||
CmiGetArgFlag(argv,"+CPBestKnown") ||
CmiGetArgFlag(argv,"+CPSteering") ||
CmiGetArgFlag(argv,"+CPMemoryAware") ||
CmiGetArgFlag(argv,"+CPSimplex") ||
CmiGetArgFlag(argv,"+CPDivideConquer") ||
CmiGetArgFlag(argv,"+CPLDBPeriod") ||
CmiGetArgFlag(argv,"+CPLDBPeriodLinear") ||
CmiGetArgFlag(argv,"+CPLDBPeriodQuadratic") ||
CmiGetArgFlag(argv,"+CPLDBPeriodOptimal") ||
CmiGetArgFlag(argv,"+CPDefaultValues") ||
CmiGetArgFlag(argv,"+CPGatherAll") ||
CmiGetArgFlag(argv,"+CPGatherMemoryUsage") ||
CmiGetArgFlag(argv,"+CPGatherUtilization") ||
CmiGetArgFlag(argv,"+CPSaveData") ||
CmiGetArgFlag(argv,"+CPNoFilterData") ||
CmiGetArgFlag(argv,"+CPLoadData") ||
CmiGetArgFlag(argv,"+CPDataFilename") )
{
CkAbort("You specified a control point command line argument, but compiled charm++ without control point support.\n");
}
#endif
}
static void _exitHandler(envelope *env)
{
DEBUGF(("exitHandler called on %d msgtype: %d\n", CkMyPe(), env->getMsgtype()));
switch(env->getMsgtype()) {
case StartExitMsg:
CkAssert(CkMyPe()==0);
if (!_CkExitFnVec.isEmpty()) {
CkExitFn fn = _CkExitFnVec.deq();
fn();
break;
}
// else goto next
case ExitMsg:
CkAssert(CkMyPe()==0);
if(_exitStarted) {
CmiFree(env);
return;
}
_exitStarted = 1;
CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
env->setMsgtype(ReqStatMsg);
env->setSrcPe(CkMyPe());
// if exit in ring, instead of broadcasting, send in ring
if (_ringexit){
DEBUGF(("[%d] Ring Exit \n",CkMyPe()));
const int stride = CkNumPes()/_ringtoken;
int pe = 0;
while (pe<CkNumPes()) {
CmiSyncSend(pe, env->getTotalsize(), (char *)env);
pe += stride;
}
CmiFree(env);
}else{
CmiSyncBroadcastAllAndFree(env->getTotalsize(), (char *)env);
}
break;
case ReqStatMsg:
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
_messageLoggingExit();
#endif
DEBUGF(("ReqStatMsg on %d\n", CkMyPe()));
CkNumberHandler(_charmHandlerIdx,(CmiHandler)_discardHandler);
CkNumberHandler(_bocHandlerIdx, (CmiHandler)_discardHandler);
/*FAULT_EVAC*/
if(CmiNodeAlive(CkMyPe())){
#if CMK_WITH_STATS
_sendStats();
#endif
_mainDone = 1; // This is needed because the destructors for
// readonly variables will be called when the program
// exits. If the destructor is called while _mainDone
// is 0, it will assume that the readonly variable was
// declared locally. On all processors other than 0,
// _mainDone is never set to 1 before the program exits.
#if CMK_TRACE_ENABLED
if (_ringexit) traceClose();
#endif
}
if (_ringexit) {
int stride = CkNumPes()/_ringtoken;
int pe = CkMyPe()+1;
if (pe < CkNumPes() && pe % stride != 0)
CmiSyncSendAndFree(pe, env->getTotalsize(), (char *)env);
else
CmiFree(env);
}
else
CmiFree(env);
//everyone exits here - there may be issues with leftover messages in the queue
#if CMK_WITH_STATS
if(CkMyPe())
#endif
{
DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
ConverseExit();
if(CharmLibInterOperate)
CpvAccess(interopExitFlag) = 1;
}
break;
#if CMK_WITH_STATS
case StatMsg:
CkAssert(CkMyPe()==0);
_allStats[env->getSrcPe()] = (Stats*) EnvToUsr(env);
_numStatsRecd++;
DEBUGF(("StatMsg on %d with %d\n", CkMyPe(), _numStatsRecd));
/*FAULT_EVAC*/
if(_numStatsRecd==CkNumValidPes()) {
_printStats();
DEBUGF(("[%d] Calling converse exit \n",CkMyPe()));
ConverseExit();
if(CharmLibInterOperate)
CpvAccess(interopExitFlag) = 1;
}
break;
#endif
default:
CmiAbort("Internal Error(_exitHandler): Unknown-msg-type. Contact Developers.\n");
}
}
void CcsInit(char **argv)
{
CpvInitialize(CkHashtable_c, ccsTab);
CpvAccess(ccsTab) = CkCreateHashtable_string(sizeof(CcsHandlerRec),5);
CpvInitialize(CcsImplHeader *, ccsReq);
CpvAccess(ccsReq) = NULL;
_ccsHandlerIdx = CmiRegisterHandler((CmiHandler)req_fw_handler);
#if CMK_BIGSIM_CHARM
CpvInitialize(int, _bgCcsHandlerIdx);
CpvAccess(_bgCcsHandlerIdx) = 0;
CpvInitialize(int, _bgCcsAck);
CpvAccess(_bgCcsAck) = 0;
#endif
CpvInitialize(int, cmiArgDebugFlag);
CpvInitialize(char *, displayArgument);
CpvInitialize(int, cpdSuspendStartup);
CpvAccess(cmiArgDebugFlag) = 0;
CpvAccess(displayArgument) = NULL;
CpvAccess(cpdSuspendStartup) = 0;
CcsBuiltinsInit(argv);
rep_fw_handler_idx = CmiRegisterHandler((CmiHandler)rep_fw_handler);
#if NODE_0_IS_CONVHOST
#if ! CMK_CMIPRINTF_IS_A_BUILTIN
print_fw_handler_idx = CmiRegisterHandler((CmiHandler)print_fw_handler);
#endif
{
int ccs_serverPort=0;
char *ccs_serverAuth=NULL;
if (CmiGetArgFlagDesc(argv,"++server", "Create a CCS server port") |
CmiGetArgIntDesc(argv,"++server-port",&ccs_serverPort, "Listen on this TCP/IP port number") |
CmiGetArgStringDesc(argv,"++server-auth",&ccs_serverAuth, "Use this CCS authentication file"))
if (CmiMyPe()==0)
{/*Create and occasionally poll on a CCS server port*/
CcsServer_new(NULL,&ccs_serverPort,ccs_serverAuth);
CcdCallOnConditionKeep(CcdPERIODIC,(CcdVoidFn)CcsServerCheck,NULL);
}
}
#endif
/* if in parallel debug mode i.e ++cpd, freeze */
if (CmiGetArgFlagDesc(argv, "+cpd", "Used *only* in conjunction with parallel debugger"))
{
CpvAccess(cmiArgDebugFlag) = 1;
if (CmiGetArgStringDesc(argv, "+DebugDisplay",&(CpvAccess(displayArgument)), "X display for gdb used only in cpd mode"))
{
if (CpvAccess(displayArgument) == NULL)
CmiPrintf("WARNING> NULL parameter for +DebugDisplay\n***");
}
else if (CmiMyPe() == 0)
{
/* only one processor prints the warning */
CmiPrintf("WARNING> x term for gdb needs to be specified as +DebugDisplay by debugger\n***\n");
}
if (CmiGetArgFlagDesc(argv, "+DebugSuspend", "Suspend execution at beginning of program")) {
CpvAccess(cpdSuspendStartup) = 1;
}
}
CcsReleaseMessages();
}
CcsHandlerRec *CcsGetHandler(const char *name) {
return CkHashtableGet(CpvAccess(ccsTab),(void *)&name);
}
void CcsRegisterHandler(const char *name, CmiHandler fn) {
CcsHandlerRec cp;
initHandlerRec(&cp,name);
cp.fnOld=fn;
*(CcsHandlerRec *)CkHashtablePut(CpvAccess(ccsTab),(void *)&cp.name)=cp;
}
void CcsSendReplyNoError(int replyLen, const void *replyData) {
if (CpvAccess(ccsReq)==NULL) return;
CcsSendReply(replyLen, replyData);
}
void CcsCallerId(skt_ip_t *pip, unsigned int *pport)
{
*pip = CpvAccess(ccsReq)->attr.ip;
*pport = ChMessageInt(CpvAccess(ccsReq)->attr.port);
}
//! process command line arguments!
void TraceCounter::traceInit(char **argv)
{
CpvInitialize(CountLogPool*, _logPool);
CpvInitialize(char*, _logName);
CpvInitialize(double, version);
CpvInitialize(char**, _counterNames);
CpvInitialize(char**, _counterDesc);
CpvInitialize(int, _numCounters);
CpvInitialize(int, _reductionID);
CpvAccess(_logName) = (char *) malloc(strlen(argv[0])+1);
_MEMCHECK(CpvAccess(_logName));
strcpy(CpvAccess(_logName), argv[0]);
CpvAccess(version) = VER;
int i;
// parse command line args
char* counters = NULL;
commandLine_ = NULL;
bool badArg = false;
int numCounters = 0;
if (CmiGetArgStringDesc(argv, "+counters", &counters, "Measure these performance counters")) {
if (CmiMyPe()==0) { CmiPrintf("Counters: %s\n", counters); }
int offset = 0;
int limit = strlen(counters);
char* ptr = counters;
while (offset < limit &&
(ptr = strtok(&counters[offset], ",")) != NULL)
{
offset += strlen(ptr)+1;
ptr = &ptr[strlen(ptr)+1];
numCounters++;
}
if (CmiMyPe()==0) {
CmiPrintf("There are %d counters\n", numCounters);
}
commandLine_ = new CounterArg[numCounters];
ptr = counters;
for (i=0; i<numCounters; i++) {
commandLine_[i].arg = ptr;
if (!matchArg(&commandLine_[i])) {
if (CmiMyPe()==0) { CmiPrintf("Bad arg: [%s]\n", ptr); }
badArg = true;
}
ptr = &ptr[strlen(ptr)+1];
}
}
commandLineSz_ = numCounters;
// check to see if args are valid, output if not
if (badArg || CmiGetArgFlagDesc(argv, "+count-help", "List available performance counters")) {
if (CmiMyPe() == 0) { printHelp(); }
ConverseExit(); return;
}
else if (counters == NULL) {
if (CmiMyPe() == 0) { usage(); }
ConverseExit(); return;
}
// get optional command line args
overview_ = CmiGetArgFlag(argv, "+count-overview");
switchRandom_ = CmiGetArgFlag(argv, "+count-switchrandom");
switchByPhase_ = CmiGetArgFlag(argv, "+count-switchbyphase");
noLog_ = CmiGetArgFlag(argv, "+count-nolog");
writeByPhase_ = CmiGetArgFlag(argv, "+count-writebyphase");
char* logName = NULL;
if (CmiGetArgString(argv, "+count-logname", &logName)) {
CpvAccess(_logName) = logName;
if (noLog_) {
if (CkMyPe()==0) {
CmiPrintf("+count-logname and +count-nolog are MUTUALLY EXCLUSIVE\n");
usage();
CmiAbort("");
}
}
}
if (switchByPhase_ && overview_) {
if (CkMyPe()==0) {
CmiPrintf(
"+count-switchbyphase and +count-overview are MUTUALLY EXCLUSIVE\n"
"+count-overview automatically switches by phase.\n");
usage();
CmiAbort("");
}
}
if (writeByPhase_ && noLog_) {
if (CkMyPe()==0) {
CmiPrintf("+count-writebyphase and +count-nolog are MUTUALLY EXCLUSIVE\n");
usage();
CmiAbort("");
}
}
// parse through commandLine_, figure out which belongs on which list (1 vs 2)
CounterArg* last1 = NULL;
CounterArg* last2 = NULL;
CounterArg* tmp = NULL;
counter1Sz_ = counter2Sz_ = 0;
for (i=0; i<commandLineSz_; i++) {
tmp = &commandLine_[i];
if (tmp->code < NUM_COUNTER_ARGS/2) {
if (counter1_ == NULL) { counter1_ = tmp; last1 = counter1_; }
else { last1->next = tmp; last1 = tmp; }
counter1Sz_++;
}
else {
if (counter2_ == NULL) { counter2_ = tmp; last2 = counter2_; }
else { last2->next = tmp; last2 = tmp; }
counter2Sz_++;
}
}
if (counter1_ == NULL) {
printHelp();
if (CmiMyPe()==0) {
CmiPrintf("\nMust specify some counters with code < %d\n",
NUM_COUNTER_ARGS/2);
}
ConverseExit();
}
if (counter2_ == NULL) {
printHelp();
if (CmiMyPe()==0) {
CmiPrintf("\nMust specify some counters with code >= %d\n",
NUM_COUNTER_ARGS/2);
}
ConverseExit();
}
last1->next = counter1_;
last2->next = counter2_;
// all args valid, now set up logging
if (CmiMyPe() == 0) {
CmiPrintf("Running with tracemode=counter and args:\n");
// print out counter1 set
tmp = counter1_;
i = 0;
do {
CmiPrintf(" <counter1-%d>=%d %s %s\n", i, tmp->code, tmp->arg, tmp->desc);
tmp = tmp->next;
i++;
} while (tmp != counter1_);
// print out counter2 set
tmp = counter2_;
i = 0;
do {
CmiPrintf(" <counter2-%d>=%d %s %s\n", i, tmp->code, tmp->arg, tmp->desc);
tmp = tmp->next;
i++;
} while (tmp != counter2_);
CmiPrintf(
"+count-overview %d\n+count-switchrandom %d\n"
"+count-switchbyphase %d\n+count-nolog %d\n"
"+count-logname %s\n+count-writebyphase %d\n",
overview_, switchRandom_, switchByPhase_, noLog_,
logName, writeByPhase_);
}
// DEBUGF((" DEBUG: Counter1=%d Counter2=%d\n", counter1_, counter2_));
CpvAccess(_logPool) = new CountLogPool();
// allocate names so can do reduction/analysis on the fly
char** counterNames = new char*[counter1Sz_+counter2Sz_];
char** counterDesc = new char*[counter1Sz_+counter2Sz_];
tmp = counter1_;
for (i=0; i<counter1Sz_; i++) {
tmp->index = i;
counterNames[i] = tmp->arg;
counterDesc[i] = tmp->desc;
tmp = tmp->next;
}
tmp = counter2_;
for (i=0; i<counter2Sz_; i++) {
tmp->index = counter1Sz_+i;
counterNames[counter1Sz_+i] = tmp->arg;
counterDesc[counter1Sz_+i] = tmp->desc;
tmp = tmp->next;
}
CpvAccess(_counterNames) = counterNames;
CpvAccess(_counterDesc) = counterDesc;
CpvAccess(_numCounters) = numCounters;
// don't erase counterNames or counterDesc,
// the reduction client will do it on the final reduction
_MEMCHECK(CpvAccess(_logPool));
CpvAccess(_logPool)->init(numCounters);
DEBUGF(("%d/%d DEBUG: Created _logPool at %08x\n",
CmiMyPe(), CmiNumPes(), CpvAccess(_logPool)));
}
/**
This is the main charm setup routine. It's called
on all processors after Converse initialization.
This routine gets passed to Converse from "main.C".
The main purpose of this routine is to set up the objects
and Ckpv's used during a regular Charm run. See the comment
at the top of the file for overall flow.
*/
void _initCharm(int unused_argc, char **argv)
{
int inCommThread = (CmiMyRank() == CmiMyNodeSize());
DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer()));
CkpvInitialize(size_t *, _offsets);
CkpvAccess(_offsets) = new size_t[32];
CkpvInitialize(PtrQ*,_buffQ);
CkpvInitialize(PtrVec*,_bocInitVec);
CkpvInitialize(void*, _currentChare);
CkpvInitialize(int, _currentChareType);
CkpvInitialize(CkGroupID, _currentGroup);
CkpvInitialize(void *, _currentNodeGroupObj);
CkpvInitialize(CkGroupID, _currentGroupRednMgr);
CkpvInitialize(GroupTable*, _groupTable);
CkpvInitialize(GroupIDTable*, _groupIDTable);
CkpvInitialize(CmiImmediateLockType, _groupTableImmLock);
CkpvInitialize(bool, _destroyingNodeGroup);
CkpvAccess(_destroyingNodeGroup) = false;
CkpvInitialize(UInt, _numGroups);
CkpvInitialize(int, _numInitsRecd);
CkpvInitialize(int, _initdone);
CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv;
CkpvInitialize(MsgPool*, _msgPool);
CkpvInitialize(CkCoreState *, _coreState);
/*
Added for evacuation-sayantan
*/
#ifndef __BIGSIM__
CpvInitialize(char *,_validProcessors);
#endif
CkpvInitialize(char ,startedEvac);
CpvInitialize(int,serializer);
_initChareTables(); // for checkpointable plain chares
CksvInitialize(UInt, _numNodeGroups);
CksvInitialize(GroupTable*, _nodeGroupTable);
CksvInitialize(GroupIDTable, _nodeGroupIDTable);
CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock);
CksvInitialize(CmiNodeLock, _nodeLock);
CksvInitialize(PtrVec*,_nodeBocInitVec);
CksvInitialize(UInt,_numInitNodeMsgs);
CkpvInitialize(int,_charmEpoch);
CkpvAccess(_charmEpoch)=0;
CksvInitialize(int, _triggersSent);
CksvAccess(_triggersSent) = 0;
CkpvInitialize(_CkOutStream*, _ckout);
CkpvInitialize(_CkErrStream*, _ckerr);
CkpvInitialize(Stats*, _myStats);
CkpvAccess(_groupIDTable) = new GroupIDTable(0);
CkpvAccess(_groupTable) = new GroupTable;
CkpvAccess(_groupTable)->init();
CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock();
CkpvAccess(_numGroups) = 1; // make 0 an invalid group number
CkpvAccess(_buffQ) = new PtrQ();
CkpvAccess(_bocInitVec) = new PtrVec();
CkpvAccess(_currentNodeGroupObj) = NULL;
if(CkMyRank()==0)
{
CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number
CksvAccess(_numInitNodeMsgs) = 0;
CksvAccess(_nodeLock) = CmiCreateLock();
CksvAccess(_nodeGroupTable) = new GroupTable();
CksvAccess(_nodeGroupTable)->init();
CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock();
CksvAccess(_nodeBocInitVec) = new PtrVec();
}
CkCallbackInit();
CmiNodeAllBarrier();
#if ! CMK_BIGSIM_CHARM
initQd(argv); // bigsim calls it in ConverseCommonInit
#endif
CkpvAccess(_coreState)=new CkCoreState();
CkpvAccess(_numInitsRecd) = 0;
CkpvAccess(_initdone) = 0;
CkpvAccess(_ckout) = new _CkOutStream();
CkpvAccess(_ckerr) = new _CkErrStream();
_charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler);
_initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
_roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler);
_exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler);
//added for interoperabilitY
_libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler);
_bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler);
CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState));
#ifdef __BIGSIM__
if(BgNodeRank()==0)
#endif
_infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn);
_triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler);
_ckModuleInit();
CldRegisterEstimator((CldEstimator)_charmLoadEstimator);
_futuresModuleInit(); // part of futures implementation is a converse module
_loadbalancerInit();
_metabalancerInit();
#if CMK_MEM_CHECKPOINT
init_memcheckpt(argv);
#endif
initCharmProjections();
#if CMK_TRACE_IN_CHARM
// initialize trace module in ck
traceCharmInit(argv);
#endif
CkpvInitialize(int, envelopeEventID);
CkpvAccess(envelopeEventID) = 0;
CkMessageWatcherInit(argv,CkpvAccess(_coreState));
/**
The rank-0 processor of each node calls the
translator-generated "_register" routines.
_register routines call the charm.h "CkRegister*" routines,
which record function pointers and class information for
all Charm entities, like Chares, Arrays, and readonlies.
There's one _register routine generated for each
.ci file. _register routines *must* be called in the
same order on every node, and *must not* be called by
multiple threads simultaniously.
*/
#ifdef __BIGSIM__
if(BgNodeRank()==0)
#else
if(CkMyRank()==0)
#endif
{
SDAG::registerPUPables();
CmiArgGroup("Charm++",NULL);
_parseCommandLineOpts(argv);
_registerInit();
CkRegisterMsg("System", 0, 0, CkFreeMsg, sizeof(int));
CkRegisterChareInCharm(CkRegisterChare("null", 0, TypeChare));
CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare), TypeChare);
CkRegisterChareInCharm(CkIndex_Chare::__idx);
CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group), TypeGroup);
CkRegisterChareInCharm(CkIndex_Group::__idx);
CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC);
/**
These _register calls are for the built-in
Charm .ci files, like arrays and load balancing.
If you add a .ci file to charm, you'll have to
add a call to the _register routine here, or make
your library into a "-module".
*/
_registerCkFutures();
_registerCkArray();
_registerLBDatabase();
_registerMetaBalancer();
_registerCkCallback();
_registertempo();
_registerwaitqd();
_registerCkCheckpoint();
#if CMK_MEM_CHECKPOINT
_registerCkMemCheckpoint();
#endif
/*
Setup Control Point Automatic Tuning Framework.
By default it is enabled as a part of charm,
however it won't enable its tracing module
unless a +CPEnableMeasurements command line argument
is specified. See trace-common.C for more info
Thus there should be no noticable overhead to
always having the control point framework linked
in.
*/
#if CMK_WITH_CONTROLPOINT
_registerPathHistory();
_registerControlPoints();
_registerTraceControlPoints();
#endif
/**
CkRegisterMainModule is generated by the (unique)
"mainmodule" .ci file. It will include calls to
register all the .ci files.
*/
CkRegisterMainModule();
/**
_registerExternalModules is actually generated by
charmc at link time (as "moduleinit<pid>.C").
This generated routine calls the _register functions
for the .ci files of libraries linked using "-module".
This funny initialization is most useful for AMPI/FEM
programs, which don't have a .ci file and hence have
no other way to control the _register process.
*/
_registerExternalModules(argv);
_registerDone();
}
/* The following will happen on every virtual processor in BigEmulator, not just on once per real processor */
if (CkMyRank() == 0) {
CpdBreakPointInit();
}
CmiNodeAllBarrier();
// Execute the initcalls registered in modules
_initCallTable.enumerateInitCalls();
#if CMK_CHARMDEBUG
CpdFinishInitialization();
#endif
//CmiNodeAllBarrier();
CkpvAccess(_myStats) = new Stats();
CkpvAccess(_msgPool) = new MsgPool();
CmiNodeAllBarrier();
#if !(__FAULT__)
CmiBarrier();
CmiBarrier();
CmiBarrier();
#endif
#if CMK_SMP_TRACE_COMMTHREAD
_TRACE_BEGIN_COMPUTATION();
#else
if (!inCommThread) {
_TRACE_BEGIN_COMPUTATION();
}
#endif
#ifdef ADAPT_SCHED_MEM
if(CkMyRank()==0){
memCriticalEntries = new int[numMemCriticalEntries];
int memcnt=0;
for(int i=0; i<_entryTable.size(); i++){
if(_entryTable[i]->isMemCritical){
memCriticalEntries[memcnt++] = i;
}
}
}
#endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
_messageLoggingInit();
#endif
#ifndef __BIGSIM__
/*
FAULT_EVAC
*/
CpvAccess(_validProcessors) = new char[CkNumPes()];
for(int vProc=0;vProc<CkNumPes();vProc++){
CpvAccess(_validProcessors)[vProc]=1;
}
_ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast);
_ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac);
#endif
CkpvAccess(startedEvac) = 0;
CpvAccess(serializer) = 0;
evacuate = 0;
CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CcdCallOnCondition(CcdSIGUSR2,(CcdVoidFn)CkMlogRestart,0);
#endif
if(_raiseEvac){
processRaiseEvacFile(_raiseEvacFile);
/*
if(CkMyPe() == 2){
// CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0);
CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
}
if(CkMyPe() == 3){
CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000);
}*/
}
if (CkMyRank() == 0) {
TopoManager_init();
}
CmiNodeAllBarrier();
if (!_replaySystem) {
CkFtFn faultFunc_restart = CkRestartMain;
if (faultFunc == NULL || faultFunc == faultFunc_restart) { // this is not restart from memory
// these two are blocking calls for non-bigsim
#if ! CMK_BIGSIM_CHARM
CmiInitCPUAffinity(argv);
CmiInitMemAffinity(argv);
#endif
}
CmiInitCPUTopology(argv);
#if CMK_SHARED_VARS_POSIX_THREADS_SMP
if (CmiCpuTopologyEnabled()) {
int *pelist;
int num;
CmiGetPesOnPhysicalNode(0, &pelist, &num);
#if !CMK_MULTICORE && !CMK_SMP_NO_COMMTHD
// Count communication threads, if present
// XXX: Assuming uniformity of node size here
num += num/CmiMyNodeSize();
#endif
if (!_Cmi_forceSpinOnIdle && num > CmiNumCores())
{
if (CmiMyPe() == 0)
CmiPrintf("\nCharm++> Warning: the number of SMP threads (%d) is greater than the number of physical cores (%d), so threads will sleep while idling. Use +CmiSpinOnIdle or +CmiSleepOnIdle to control this directly.\n\n", num, CmiNumCores());
CmiLock(CksvAccess(_nodeLock));
if (! _Cmi_sleepOnIdle) _Cmi_sleepOnIdle = 1;
CmiUnlock(CksvAccess(_nodeLock));
}
}
#endif
}
if(CmiMyPe() == 0) {
char *topoFilename;
if(CmiGetArgStringDesc(argv,"+printTopo",&topoFilename,"topo file name"))
{
std::stringstream sstm;
sstm << topoFilename << "." << CmiMyPartition();
std::string result = sstm.str();
FILE *fp;
fp = fopen(result.c_str(), "w");
if (fp == NULL) {
CkPrintf("Error opening %s file, writing to stdout\n", topoFilename);
fp = stdout;
}
TopoManager_printAllocation(fp);
fclose(fp);
}
}
#if CMK_USE_PXSHM && ( CMK_CRAYXE || CMK_CRAYXC ) && CMK_SMP
// for SMP on Cray XE6 (hopper) it seems pxshm has to be initialized
// again after cpuaffinity is done
if (CkMyRank() == 0) {
CmiInitPxshm(argv);
}
CmiNodeAllBarrier();
#endif
//CldCallback();
#if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG
// Register the BG handler for CCS. Notice that this is put into a variable shared by
// the whole real processor. This because converse needs to find it. We check that all
// virtual processors register the same index for this handler.
CpdBgInit();
#endif
if (faultFunc) {
#if CMK_WITH_STATS
if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()];
#endif
if (!inCommThread) {
CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
msg->argc = CmiGetArgc(argv);
msg->argv = argv;
faultFunc(_restartDir, msg);
CkFreeMsg(msg);
}
}else if(CkMyPe()==0){
#if CMK_WITH_STATS
_allStats = new Stats*[CkNumPes()];
#endif
register size_t i, nMains=_mainTable.size();
for(i=0;i<nMains;i++) /* Create all mainchares */
{
register int size = _chareTable[_mainTable[i]->chareIdx]->size;
register void *obj = malloc(size);
_MEMCHECK(obj);
_mainTable[i]->setObj(obj);
CkpvAccess(_currentChare) = obj;
CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx;
register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0);
msg->argc = CmiGetArgc(argv);
msg->argv = argv;
_entryTable[_mainTable[i]->entryIdx]->call(msg, obj);
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
CpvAccess(_currentObj) = (Chare *)obj;
#endif
}
_mainDone = 1;
_STATS_RECORD_CREATE_CHARE_N(nMains);
_STATS_RECORD_PROCESS_CHARE_N(nMains);
for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */
{
register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg));
if(roMsg==0)
continue;
//Pack the message and send it to all other processors
register envelope *env = UsrToEnv(roMsg);
env->setSrcPe(CkMyPe());
env->setMsgtype(ROMsgMsg);
env->setRoIdx(i);
CmiSetHandler(env, _initHandlerIdx);
CkPackMessage(&env);
CmiSyncBroadcast(env->getTotalsize(), (char *)env);
CpvAccess(_qd)->create(CkNumPes()-1);
//For processor 0, unpack and re-set the global
CkUnpackMessage(&env);
_processROMsgMsg(env);
_numInitMsgs++;
}
//Determine the size of the RODataMessage
PUP::sizer ps;
for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps);
//Allocate and fill out the RODataMessage
envelope *env = _allocEnv(RODataMsg, ps.size());
PUP::toMem pp((char *)EnvToUsr(env));
for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp);
env->setCount(++_numInitMsgs);
env->setSrcPe(CkMyPe());
CmiSetHandler(env, _initHandlerIdx);
DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize()));
CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env);
CpvAccess(_qd)->create(CkNumPes()-1);
_initDone();
}
DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread));
// when I am a communication thread, I don't participate initDone.
if (inCommThread) {
CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler,
CkpvAccess(_coreState));
CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler
,
CkpvAccess(_coreState));
_processBufferedMsgs();
}
#if CMK_CHARMDEBUG
// Should not use CpdFreeze inside a thread (since this processor is really a user-level thread)
if (CpvAccess(cpdSuspendStartup))
{
//CmiPrintf("In Parallel Debugging mode .....\n");
CpdFreeze();
}
#endif
#if __FAULT__
if(killFlag){
readKillFile();
}
#endif
}
//! end computation, do a reduction here in hopes that it finishes before
//! traceClose called and the program exits
void TraceCounter::endComputation() {
CpvAccess(_logPool)->doReduction(-1, idleTime_);
}
/// Basic Constructor
PVT::PVT()
{
#ifdef VERBOSE_DEBUG
CkPrintf("[%d] constructing PVT\n",CkMyPe());
#endif
CpvInitialize(int, stateRecovery);
CpvAccess(stateRecovery) = 0;
CpvInitialize(eventID, theEventID);
CpvAccess(theEventID)=eventID();
// CpvAccess(theEventID).dump();
//LBTurnInstrumentOff();
optGVT = POSE_UnsetTS; conGVT = POSE_UnsetTS;
rdone=0;
SRs=NULL;
#ifdef POSE_COMM_ON
//com_debug = 1;
#endif
#ifndef CMK_OPTIMIZE
localStats = (localStat *)CkLocalBranch(theLocalStats);
if (pose_config.stats) {
localStats->TimerStart(GVT_TIMER);
}
#endif
#ifdef MEM_TEMPORAL
localTimePool = (TimePool *)CkLocalBranch(TempMemID);
CkPrintf("NOTE: Temporal memory manager is ON!\n");
#endif
optPVT = conPVT = estGVT = POSE_UnsetTS;
startPhaseActive = gvtTurn = simdone = 0;
SendsAndRecvs = new SRtable();
SendsAndRecvs->Initialize();
specEventCount = eventCount = waitForFirst = 0;
iterMin = POSE_UnsetTS;
int P=CkNumPes(), N=CkMyPe();
reportReduceTo = -1;
if ((N < P-2) && (N%2 == 1)) { //odd
reportTo = N-1;
reportsExpected = reportEnd = 0;
}
else if (N < P-2) { //even
reportTo = N;
reportsExpected = 2;
if (N == P-3)
reportsExpected = 1;
reportEnd = 0;
if (N < (P-2)/2)
reportReduceTo = P-2;
else reportReduceTo = P-1;
}
if (N == P-2) {
reportTo = N;
reportEnd = 1;
reportsExpected = 1 + (P-2)/4 + ((P-2)%4)/2;
}
else if (N == P-1) {
reportTo = N;
reportEnd = 1;
if (P==1) reportsExpected = 1;
else reportsExpected = 1 + (P-2)/4 + (P-2)%2;
}
// CkPrintf("PE %d reports to %d, receives %d reports, reduces and sends to %d, and reports directly to GVT if %d = 1!\n", CkMyPe(), reportTo, reportsExpected, reportReduceTo, reportEnd);
parCheckpointInProgress = 0;
parLastCheckpointGVT = 0;
parLastCheckpointTime = parStartTime = 0.0;
parLBInProgress = 0;
parLastLBGVT = 0;
// debugBufferLoc = debugBufferWrapped = debugBufferDumped = 0;
#ifndef CMK_OPTIMIZE
if(pose_config.stats)
localStats->TimerStop();
#endif
LBDatabase::Object()->AddMigrationDoneFn(staticDoneLB, this);
}
//! write the summary sts file for this trace
void TraceCounter::traceWriteSts() {
if (traceOn_ && !noLog_) {
if (CmiMyPe()==0) { CpvAccess(_logPool)->writeSts(); }
}
}
void resumeCentralLbAfterChkpt(void *_lb){
CentralLB *lb= (CentralLB *)_lb;
CpvAccess(_currentObj)=lb;
lb->endMigrationDone(lb->savedBalancing);
}
int pvmc_unpackmsg(void *msgbuf, void *start_of_msg)
{
pvmc_buffer *cur_buf;
pvmc_item *cur_item, *nxt_item;
int bytes_unpacked=0;
int i;
if ((CpvAccess(pvmc_rbufid)<=0) || (CpvAccess(pvmc_rbufid) >= MAX_BUFFERS) ||
(CpvAccess(pvmc_bufarray)[CpvAccess(pvmc_rbufid)].refcount <= 0)) {
PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() uninitialized recv buffer\n",
MYPE(),__FILE__,__LINE__);
return -1;
}
cur_buf = &(CpvAccess(pvmc_bufarray)[CpvAccess(pvmc_rbufid)]);
pvmc_emptybuf(cur_buf);
cur_buf->bytes = *((int *)((char *)start_of_msg+bytes_unpacked));
bytes_unpacked += sizeof(int);
cur_buf->tag = *((int *)((char *)start_of_msg+bytes_unpacked));
bytes_unpacked += sizeof(int);
cur_buf->tid = *((int *)((char *)start_of_msg+bytes_unpacked));
bytes_unpacked += sizeof(int);
cur_buf->num_items = *((int *)((char *)start_of_msg+bytes_unpacked));
bytes_unpacked += sizeof(int);
#ifdef PVM_DEBUG
PRINTF("Pe(%d) pvmc_unpackmsg: %d items unpacked for tag %d\n",
MYPE(),cur_buf->num_items,cur_buf->tag);
#endif
if (msgbuf)
cur_buf->data_buf = msgbuf;
else cur_buf->data_buf = (void *)NULL;
cur_item=(pvmc_item *)MALLOC(sizeof(pvmc_item));
cur_buf->first_item=cur_item;
cur_buf->cur_item=cur_item;
if (cur_item==(pvmc_item *)NULL) {
PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() can't allocate memory\n",
MYPE(),pvm_mytid(),__FILE__,__LINE__);
return -1;
}
#if PVM_DEBUG
PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() unpacking %d messages.\n",
MYPE(),pvm_mytid(),__FILE__,__LINE__,cur_buf->num_items);
#endif
for(i=0;i<cur_buf->num_items;i++) {
cur_item->type = *((int *)((char *)start_of_msg+bytes_unpacked));
bytes_unpacked+=sizeof(int);
cur_item->size = *((int *)((char *)start_of_msg+bytes_unpacked));
bytes_unpacked+=sizeof(int);
nxt_item=(pvmc_item *)MALLOC(sizeof(pvmc_item));
if (!nxt_item) {
PRINTF("Pe(%d) tid=%d:%s:%d pvmc_unpackmsg() can't allocate memory\n",
MYPE(),pvm_mytid(),__FILE__,__LINE__);
return -1;
}
cur_item->nxt = nxt_item;
cur_item = nxt_item;
}
cur_item->type = 0;
cur_item->size = 0;
cur_item->free_data = FALSE;
cur_item->data = (char *) NULL;
cur_item->nxt = (pvmc_item *) NULL;
cur_buf->last_item = cur_item;
cur_item = cur_buf->first_item;
while(cur_item!=cur_buf->last_item) {
if (cur_item->size > 0) {
cur_item->free_data=FALSE;
cur_item->data = (void *)((char *)start_of_msg+bytes_unpacked);
bytes_unpacked+=cur_item->size;
}
else cur_item->data=NULL;
cur_item = cur_item->nxt;
}
return bytes_unpacked;
}
void CpdUnFreeze(void)
{
CpvAccess(freezeModeFlag) = 0;
}
CmiStartFn mymain(int argc, char** argv)
{
if(CmiMyRank() == CmiMyNodeSize()) return 0;
CpvInitialize(int,msgSize);
CpvInitialize(int,cycleNum);
CpvInitialize(int,sizeNum);
CpvAccess(sizeNum) = 1;
CpvAccess(msgSize)= CmiMsgHeaderSizeBytes + 8;
CpvInitialize(int,exitHandler);
CpvAccess(exitHandler) = CmiRegisterHandler((CmiHandler) exitHandlerFunc);
CpvInitialize(int,node0Handler);
CpvAccess(node0Handler) = CmiRegisterHandler((CmiHandler) node0HandlerFunc);
CpvInitialize(int,node1Handler);
CpvAccess(node1Handler) = CmiRegisterHandler((CmiHandler) node1HandlerFunc);
CpvInitialize(int,ackHandler);
CpvAccess(ackHandler) = CmiRegisterHandler((CmiHandler) ackHandlerFunc);
CpvInitialize(double,startTime);
CpvInitialize(double,endTime);
CpvInitialize(double, IdleStartTime);
CpvInitialize(double, IdleTime);
CpvInitialize(int,ackCount);
CpvAccess(ackCount) = 0;
CpvInitialize(int,twoway);
CpvAccess(twoway) = 0;
CcdCallOnConditionKeep(CcdPROCESSOR_BEGIN_IDLE, ApplIdleStart, NULL);
CcdCallOnConditionKeep(CcdPROCESSOR_END_IDLE, ApplIdleEnd, NULL);
if(argc > 1)
CpvAccess(twoway) = atoi(argv[1]);
if(CmiMyPe() == 0) {
if(!CpvAccess(twoway))
CmiPrintf("Starting Pingpong with oneway traffic \n");
else
CmiPrintf("Starting Pingpong with twoway traffic\n");
}
if ((CmiMyPe() < CmiNumPes()/2) || CpvAccess(twoway))
startPingpong();
return 0;
}
int CpdIsFrozen(void) {
return CpvAccess(freezeModeFlag);
}
/**
* Register a callback function that will be triggered *whenever* the specified
* condition is raised
*/
int CcdCallOnConditionKeep(int condnum, CcdVoidFn fnp, void *arg)
{
return append_elem(&(CpvAccess(conds).condcb_keep[condnum]), fnp, arg, CcdIGNOREPE);
}
//! do a reduction across processors to calculate the total count for
//! each count, and if the count has flops, etc, then calc the
//! the flops/s, etc...
void StatTable::doReduction(int phase, double idleTime) {
DEBUGF(("%d/%d DEBUG: StatTable::doReduction()\n",
CmiMyPe(), CmiNumPes(), this));
// see above (NUM_EXTRA_PERF) for the fields in the message
int msgSize =
ALIGN8(CmiMsgHeaderSizeBytes)+
sizeof(double)*(2*numStats_+NUM_EXTRA_PERF);
char *msg = (char *)CmiAlloc(msgSize);
double* reduction = (double*)(msg+ALIGN8(CmiMsgHeaderSizeBytes));
// calculate flops/s, l1%, l2%, tlb% if it's there
char** counterNames = CpvAccess(_counterNames);
int GR_FLOPS = -1; double flopsRate = -1.0;
int LOAD = -1; double loadRate = -1.0;
int STORE = -1; double storeRate = -1.0;
int L1_DMISS = -1; double l1Rate = -1.0;
int L2_DMISS = -1; double l2Rate = -1.0;
int TLB_MISS = -1; double tlbRate = -1.0;
int i, j;
for (i=0; i<2*numStats_+NUM_EXTRA_PERF; i++) { reduction[i] = 0.0; }
for (i=0; i<numStats_; i++) {
for (int j=0; j<MAX_ENTRIES; j++) {
reduction[2*i] += stats_[i].numCalled[j]*stats_[i].avgCount[j];
reduction[2*i+1] += stats_[i].totTime[j];
}
if (strcmp(counterNames[i], "GR_FLOPS")==0) { GR_FLOPS = i; }
else if (strcmp(counterNames[i], "LOAD")==0) { LOAD = i; }
else if (strcmp(counterNames[i], "STORE")==0) { STORE = i; }
else if (strcmp(counterNames[i], "L1_DMISS")==0) { L1_DMISS = i; }
else if (strcmp(counterNames[i], "L2_DMISS")==0) { L2_DMISS = i; }
else if (strcmp(counterNames[i], "TLB_MISS")==0) { TLB_MISS = i; }
}
if (CmiMyPe()==0) { reduction[2*numStats_] = phase; }
reduction[2*numStats_+1] = idleTime;
// -1 for the rest of the calc values
reduction[2*numStats_+2] = -1.0;
reduction[2*numStats_+3] = -1.0;
reduction[2*numStats_+4] = -1.0;
reduction[2*numStats_+5] = -1.0;
// calculate flops/s, l1%, l2%, tlb% if it's there
double* rate = NULL;
int index;
for (i=0; i<6; i++) {
switch (i) {
case 0: rate = &flopsRate; index = GR_FLOPS; break;
case 1: rate = &loadRate; index = LOAD; break;
case 2: rate = &storeRate; index = STORE; break;
case 3: rate = &l1Rate; index = L1_DMISS; break;
case 4: rate = &l2Rate; index = L2_DMISS; break;
case 5: rate = &tlbRate; index = TLB_MISS; break;
}
if (index >= 0 && reduction[2*index+1] > 0.0) {
// if we have the counter AND it's times were non-zero
*rate = reduction[2*index]/reduction[2*index+1];
}
}
// store rates if there
if (GR_FLOPS >= 0) { reduction[2*numStats_+2] = flopsRate; }
if (LOAD >= 0 && STORE >= 0) {
double memRate = loadRate + storeRate;
if (L1_DMISS >= 0 & memRate > 0) {
reduction[2*numStats_+3] = l1Rate / memRate;
}
if (L2_DMISS >= 0 & memRate > 0) {
reduction[2*numStats_+4] = l2Rate / memRate;
}
if (TLB_MISS >= 0 & memRate > 0) {
reduction[2*numStats_+5] = tlbRate / memRate;
}
}
// send the data
CmiSetHandler(msg, (int)CpvAccess(_reductionID));
int handlerID = CmiGetHandler(msg);
DEBUGF(("%d/%d handlerID %d reductionID %d\n",
CmiMyPe(), CmiNumPes(), handlerID, CpvAccess(_reductionID)));
CmiSyncSendAndFree(0, msgSize, msg);
}
/**
* Cancel a previously registered conditional callback
*/
void CcdCancelCallOnConditionKeep(int condnum, int idx)
{
remove_elem(&(CpvAccess(conds).condcb_keep[condnum]), idx);
}
// a rudimentary reduction to print out the performance results across the run
CmiHandler StatTableReduction(char* msg)
{
DEBUGF(("StatTableReduction called\n", CmiMyPe(), CmiNumPes()));
static double* reduce = NULL;
static int numReduce = 0;
int numCounters = CpvAccess(_numCounters);
int size = 2*CpvAccess(_numCounters)+NUM_EXTRA_PERF;
int i;
if (reduce == NULL) {
// allocate
reduce = new double[size];
for (i=0; i<size; i++) { reduce[i] = 0.0; }
DEBUGF((" allocated reduce numCounters %d size %d\n",
numCounters, size));
}
// see above for the feilds of this message
double* msgResults = (double *)(msg+ALIGN8(CmiMsgHeaderSizeBytes));
for (i=0; i<size; i++) {
reduce[i] += msgResults[i];
}
char** counterNames = CpvAccess(_counterNames);
numReduce++;
DEBUGF((" numReduce %d numPes %d\n", numReduce, CmiNumPes()));
int phase = reduce[2*numCounters];
if (numReduce >= CmiNumPes()) {
// finished with reduction, print out results
numReduce = 0;
for (i=0; i<numCounters; i++) {
if (reduce[2*i+1]>0.0) { // is time > 0?
if (phase >= 0) {
CmiPrintf("PHASE %d %s totalCount %f totalTime (us) %f\n"
"PHASE %d %s count/proc %f avgTime (us)/phase %f\n",
phase, counterNames[i], reduce[2*i], reduce[2*i+1]*1e6,
phase, counterNames[i], reduce[2*i]/CmiNumPes(),
reduce[2*i+1]*1e6/CmiNumPes());
}
else {
CmiPrintf("%s totalCount %f totalTime (us) %f\n"
"%s count/proc %f avgTime (us)/phase %f\n",
counterNames[i], reduce[2*i], reduce[2*i+1]*1e6,
counterNames[i], reduce[2*i]/CmiNumPes(),
reduce[2*i+1]*1e6/CmiNumPes());
}
}
}
if (phase >= 0) {
CmiPrintf("PHASE %d totalIdleTime (us) %f avgIdleTime (us)/phase %f\n",
phase, reduce[2*numCounters+1]*1e6,
reduce[2*numCounters+1]*1e6/CmiNumPes());
}
else {
CmiPrintf("totalIdleTime (us) %f avgIdleTime (us)/phase %f\n",
reduce[2*numCounters+1]*1e6,
reduce[2*numCounters+1]*1e6/CmiNumPes());
}
if (reduce[2*numCounters+2] > 0.0) {
// we have flops
if (phase >= 0) {
CmiPrintf("PHASE %d flops/s %f flops/s/PE %f\n",
phase, reduce[2*numCounters+2],
reduce[2*numCounters+2]/CmiNumPes());
}
else {
CmiPrintf("flops/s %f flops/s/PE %f\n",
reduce[2*numCounters+2],
reduce[2*numCounters+2]/CmiNumPes());
}
}
char* missRate = NULL;
for (i=0; i<3; i++) {
switch (i) {
case 0: missRate = "l1 avg miss rate (%)"; break;
case 1: missRate = "l2 avg miss rate (%)"; break;
case 2: missRate = "tlb avg miss rate (%)"; break;
}
if (reduce[2*numCounters+3+i] >= 0.0) {
if (phase >= 0) {
CmiPrintf("PHASE %d %s %f\n",
phase, missRate, reduce[2*numCounters+3+i]/CmiNumPes()*100);
}
else {
CmiPrintf("%s %f\n",
missRate, reduce[2*numCounters+3+i]/CmiNumPes()*100);
}
}
}
// clean up
delete [] reduce;
reduce = NULL;
}
CmiFree(msg);
}
/**
* Called when something drastic changes-- restart ccd_num_checks
*/
void CcdCallBacksReset(void *ignored,double curWallTime)
{
ccd_periodic_callbacks *o=&CpvAccess(pcb);
CpvAccess(_ccd_numchecks)=o->nSkip=1;
o->lastCheck=curWallTime;
}
void CtgInit(void) {
CpvInitialize(int, CmiPICMethod);
CpvAccess(CmiPICMethod) = CMI_PIC_NOP;
}
