/**
* Obtain the number of nodes, my node id, and consuming machine layer
* specific arguments
*/
static void MachineInitForLAPI(int *argc, char ***argv, int *numNodes, int *myNodeID) {
lapi_info_t info;
char **largv = *argv;
memset(&info,0,sizeof(info));
/* Register error handler (redundant?) -- added by Chao Mei*/
info.err_hndlr = (LAPI_err_hndlr *)lapi_err_hndlr;
/* Indicates the number of completion handler threads to create */
/* The number of completion hndlr thds will affect the atomic PCQueue operations!! */
/* NOTE: num_compl_hndlr_thr is obsolete now! --Chao Mei */
/* info.num_compl_hndlr_thr = 1; */
check_lapi(LAPI_Init,(&lapiContext, &info));
/* It's a good idea to start with a fence,
because packets recv'd before a LAPI_Init are just dropped. */
check_lapi(LAPI_Gfence,(lapiContext));
check_lapi(LAPI_Qenv,(lapiContext, TASK_ID, myNodeID));
check_lapi(LAPI_Qenv,(lapiContext, NUM_TASKS, numNodes));
/* Make polling as the default mode as real apps have better perf */
CsvAccess(lapiInterruptMode) = 0;
if (CmiGetArgFlag(largv,"+poll")) CsvAccess(lapiInterruptMode) = 0;
if (CmiGetArgFlag(largv,"+nopoll")) CsvAccess(lapiInterruptMode) = 1;
check_lapi(LAPI_Senv,(lapiContext, ERROR_CHK, lapiDebugMode));
check_lapi(LAPI_Senv,(lapiContext, INTERRUPT_SET, CsvAccess(lapiInterruptMode)));
if (*myNodeID == 0) {
printf("Running lapi in interrupt mode: %d\n", CsvAccess(lapiInterruptMode));
printf("Running lapi with %d completion handler threads.\n", info.num_compl_hndlr_thr);
}
/**
* Associate PumpMsgsBegin with var "lapiHeaderHandler". Then inside Xfer calls,
* lapiHeaderHandler could be used to indicate the callback
* instead of PumpMsgsBegin --Chao Mei
*/
check_lapi(LAPI_Addr_set,(lapiContext,(void *)PumpMsgsBegin,lapiHeaderHandler));
if (CmiGetArgFlag(largv,"++debug")) { /*Pause so user has a chance to start and attach debugger*/
printf("CHARMDEBUG> Processor %d has PID %d\n",*myNodeID,getpid());
if (!CmiGetArgFlag(largv,"++debug-no-pause"))
sleep(30);
}
#if ENSURE_MSG_PAIRORDER
cmplHdlrThdLock = CmiCreateLock();
#endif
}
// called on all procs
void all_init(int argc, char **argv)
{
#if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW_H)
_set_new_handler(NAMD_new_handler);
#else
std::set_new_handler(NAMD_new_handler);
#endif
ProcessorPrivateInit();
register_exit_sched();
#ifdef NAMD_CUDA
CmiGetArgFlag(argv, "+idlepoll"); // remove +idlepoll if it's still there
cuda_getargs(argv);
argc = CmiGetArgc(argv);
#endif
_initCharm(argc, argv); // message main Chare
#ifdef NAMD_CUDA
if ( CkMyPe() < CkNumPes() ) cuda_initialize();
#endif
}
//! process command line arguments!
void TraceCounter::traceInit(char **argv)
{
CpvInitialize(CountLogPool*, _logPool);
CpvInitialize(char*, _logName);
CpvInitialize(double, version);
CpvInitialize(char**, _counterNames);
CpvInitialize(char**, _counterDesc);
CpvInitialize(int, _numCounters);
CpvInitialize(int, _reductionID);
CpvAccess(_logName) = (char *) malloc(strlen(argv[0])+1);
_MEMCHECK(CpvAccess(_logName));
strcpy(CpvAccess(_logName), argv[0]);
CpvAccess(version) = VER;
int i;
// parse command line args
char* counters = NULL;
commandLine_ = NULL;
bool badArg = false;
int numCounters = 0;
if (CmiGetArgStringDesc(argv, "+counters", &counters, "Measure these performance counters")) {
if (CmiMyPe()==0) { CmiPrintf("Counters: %s\n", counters); }
int offset = 0;
int limit = strlen(counters);
char* ptr = counters;
while (offset < limit &&
(ptr = strtok(&counters[offset], ",")) != NULL)
{
offset += strlen(ptr)+1;
ptr = &ptr[strlen(ptr)+1];
numCounters++;
}
if (CmiMyPe()==0) {
CmiPrintf("There are %d counters\n", numCounters);
}
commandLine_ = new CounterArg[numCounters];
ptr = counters;
for (i=0; i<numCounters; i++) {
commandLine_[i].arg = ptr;
if (!matchArg(&commandLine_[i])) {
if (CmiMyPe()==0) { CmiPrintf("Bad arg: [%s]\n", ptr); }
badArg = true;
}
ptr = &ptr[strlen(ptr)+1];
}
}
commandLineSz_ = numCounters;
// check to see if args are valid, output if not
if (badArg || CmiGetArgFlagDesc(argv, "+count-help", "List available performance counters")) {
if (CmiMyPe() == 0) { printHelp(); }
ConverseExit(); return;
}
else if (counters == NULL) {
if (CmiMyPe() == 0) { usage(); }
ConverseExit(); return;
}
// get optional command line args
overview_ = CmiGetArgFlag(argv, "+count-overview");
switchRandom_ = CmiGetArgFlag(argv, "+count-switchrandom");
switchByPhase_ = CmiGetArgFlag(argv, "+count-switchbyphase");
noLog_ = CmiGetArgFlag(argv, "+count-nolog");
writeByPhase_ = CmiGetArgFlag(argv, "+count-writebyphase");
char* logName = NULL;
if (CmiGetArgString(argv, "+count-logname", &logName)) {
CpvAccess(_logName) = logName;
if (noLog_) {
if (CkMyPe()==0) {
CmiPrintf("+count-logname and +count-nolog are MUTUALLY EXCLUSIVE\n");
usage();
CmiAbort("");
}
}
}
if (switchByPhase_ && overview_) {
if (CkMyPe()==0) {
CmiPrintf(
"+count-switchbyphase and +count-overview are MUTUALLY EXCLUSIVE\n"
"+count-overview automatically switches by phase.\n");
usage();
CmiAbort("");
}
}
if (writeByPhase_ && noLog_) {
if (CkMyPe()==0) {
CmiPrintf("+count-writebyphase and +count-nolog are MUTUALLY EXCLUSIVE\n");
usage();
CmiAbort("");
}
}
// parse through commandLine_, figure out which belongs on which list (1 vs 2)
CounterArg* last1 = NULL;
CounterArg* last2 = NULL;
CounterArg* tmp = NULL;
counter1Sz_ = counter2Sz_ = 0;
for (i=0; i<commandLineSz_; i++) {
tmp = &commandLine_[i];
if (tmp->code < NUM_COUNTER_ARGS/2) {
if (counter1_ == NULL) { counter1_ = tmp; last1 = counter1_; }
else { last1->next = tmp; last1 = tmp; }
counter1Sz_++;
}
else {
if (counter2_ == NULL) { counter2_ = tmp; last2 = counter2_; }
else { last2->next = tmp; last2 = tmp; }
counter2Sz_++;
}
}
if (counter1_ == NULL) {
printHelp();
if (CmiMyPe()==0) {
CmiPrintf("\nMust specify some counters with code < %d\n",
NUM_COUNTER_ARGS/2);
}
ConverseExit();
}
if (counter2_ == NULL) {
printHelp();
if (CmiMyPe()==0) {
CmiPrintf("\nMust specify some counters with code >= %d\n",
NUM_COUNTER_ARGS/2);
}
ConverseExit();
}
last1->next = counter1_;
last2->next = counter2_;
// all args valid, now set up logging
if (CmiMyPe() == 0) {
CmiPrintf("Running with tracemode=counter and args:\n");
// print out counter1 set
tmp = counter1_;
i = 0;
do {
CmiPrintf(" <counter1-%d>=%d %s %s\n", i, tmp->code, tmp->arg, tmp->desc);
tmp = tmp->next;
i++;
} while (tmp != counter1_);
// print out counter2 set
tmp = counter2_;
i = 0;
do {
CmiPrintf(" <counter2-%d>=%d %s %s\n", i, tmp->code, tmp->arg, tmp->desc);
tmp = tmp->next;
i++;
} while (tmp != counter2_);
CmiPrintf(
"+count-overview %d\n+count-switchrandom %d\n"
"+count-switchbyphase %d\n+count-nolog %d\n"
"+count-logname %s\n+count-writebyphase %d\n",
overview_, switchRandom_, switchByPhase_, noLog_,
logName, writeByPhase_);
}
// DEBUGF((" DEBUG: Counter1=%d Counter2=%d\n", counter1_, counter2_));
CpvAccess(_logPool) = new CountLogPool();
// allocate names so can do reduction/analysis on the fly
char** counterNames = new char*[counter1Sz_+counter2Sz_];
char** counterDesc = new char*[counter1Sz_+counter2Sz_];
tmp = counter1_;
for (i=0; i<counter1Sz_; i++) {
tmp->index = i;
counterNames[i] = tmp->arg;
counterDesc[i] = tmp->desc;
tmp = tmp->next;
}
tmp = counter2_;
for (i=0; i<counter2Sz_; i++) {
tmp->index = counter1Sz_+i;
counterNames[counter1Sz_+i] = tmp->arg;
counterDesc[counter1Sz_+i] = tmp->desc;
tmp = tmp->next;
}
CpvAccess(_counterNames) = counterNames;
CpvAccess(_counterDesc) = counterDesc;
CpvAccess(_numCounters) = numCounters;
// don't erase counterNames or counterDesc,
// the reduction client will do it on the final reduction
_MEMCHECK(CpvAccess(_logPool));
CpvAccess(_logPool)->init(numCounters);
DEBUGF(("%d/%d DEBUG: Created _logPool at %08x\n",
CmiMyPe(), CmiNumPes(), CpvAccess(_logPool)));
}
static inline void _parseCommandLineOpts(char **argv)
{
if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown"))
_STATS_ON(_printCS);
if (CmiGetArgFlagDesc(argv,"+ss", "Print summary statistics at shutdown"))
_STATS_ON(_printSS);
if (CmiGetArgFlagDesc(argv,"+fifo", "Default to FIFO queuing"))
_defaultQueueing = CK_QUEUEING_FIFO;
if (CmiGetArgFlagDesc(argv,"+lifo", "Default to LIFO queuing"))
_defaultQueueing = CK_QUEUEING_LIFO;
if (CmiGetArgFlagDesc(argv,"+ififo", "Default to integer-prioritized FIFO queuing"))
_defaultQueueing = CK_QUEUEING_IFIFO;
if (CmiGetArgFlagDesc(argv,"+ilifo", "Default to integer-prioritized LIFO queuing"))
_defaultQueueing = CK_QUEUEING_ILIFO;
if (CmiGetArgFlagDesc(argv,"+bfifo", "Default to bitvector-prioritized FIFO queuing"))
_defaultQueueing = CK_QUEUEING_BFIFO;
if (CmiGetArgFlagDesc(argv,"+blifo", "Default to bitvector-prioritized LIFO queuing"))
_defaultQueueing = CK_QUEUEING_BLIFO;
if (CmiGetArgFlagDesc(argv,"+objq", "Default to use object queue for every obejct"))
{
#if CMK_OBJECT_QUEUE_AVAILABLE
_defaultObjectQ = 1;
if (CkMyPe()==0)
CmiPrintf("Charm++> Create object queue for every Charm object.\n");
#else
CmiAbort("Charm++> Object queue not enabled, recompile Charm++ with CMK_OBJECT_QUEUE_AVAILABLE defined to 1.");
#endif
}
if(CmiGetArgString(argv,"+restart",&_restartDir))
faultFunc = CkRestartMain;
#if __FAULT__
if (CmiGetArgIntDesc(argv,"+restartaftercrash",&CpvAccess(_curRestartPhase),"restarting this processor after a crash")){
# if CMK_MEM_CHECKPOINT
faultFunc = CkMemRestart;
# endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
faultFunc = CkMlogRestart;
#endif
CmiPrintf("[%d] Restarting after crash \n",CmiMyPe());
}
#if CMK_MESSAGE_LOGGING
// reading +ftc_disk flag
if (CmiGetArgFlagDesc(argv, "+ftc_disk", "Disk Checkpointing")) {
diskCkptFlag = 1;
}
#endif
// reading the killFile
if(CmiGetArgStringDesc(argv,"+killFile", &killFile,"Generates SIGKILL on specified processors")){
if(faultFunc == NULL){
//do not read the killfile if this is a restarting processor
killFlag = 1;
if(CmiMyPe() == 0){
printf("[%d] killFlag set to 1 for file %s\n",CkMyPe(),killFile);
}
}
}
#endif
// shut down program in ring fashion to allow projections output w/o IO error
if (CmiGetArgIntDesc(argv,"+ringexit",&_ringtoken, "Program exits in a ring fashion"))
{
_ringexit = 1;
if (CkMyPe()==0)
CkPrintf("Charm++> Program shutdown in token ring (%d).\n", _ringtoken);
if (_ringtoken > CkNumPes()) _ringtoken = CkNumPes();
}
/*
FAULT_EVAC
if the argument +raiseevac is present then cause faults
*/
if(CmiGetArgStringDesc(argv,"+raiseevac", &_raiseEvacFile,"Generates processor evacuation on random processors")){
_raiseEvac = 1;
}
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
if(!CmiGetArgIntDesc(argv,"+teamSize",&teamSize,"Set the team size for message logging")){
teamSize = 1;
}
if(!CmiGetArgIntDesc(argv,"+chkptPeriod",&chkptPeriod,"Set the checkpoint period for the message logging fault tolerance algorithm in seconds")){
chkptPeriod = 100;
}
if(CmiGetArgIntDesc(argv,"+fastRecovery", ¶llelRecovery, "Parallel recovery with message logging protocol")){
fastRecovery = true;
}
#endif
/* Anytime migration flag */
_isAnytimeMigration = true;
if (CmiGetArgFlagDesc(argv,"+noAnytimeMigration","The program does not require support for anytime migration")) {
_isAnytimeMigration = false;
}
_isNotifyChildInRed = true;
if (CmiGetArgFlagDesc(argv,"+noNotifyChildInReduction","The program has at least one element per processor for each charm array created")) {
_isNotifyChildInRed = false;
}
_isStaticInsertion = false;
if (CmiGetArgFlagDesc(argv,"+staticInsertion","Array elements are only inserted at construction")) {
_isStaticInsertion = true;
}
useNodeBlkMapping = false;
if (CmiGetArgFlagDesc(argv,"+useNodeBlkMapping","Array elements are block-mapped in SMP-node level")) {
useNodeBlkMapping = true;
}
#if ! CMK_WITH_CONTROLPOINT
// Display a warning if charm++ wasn't compiled with control point support but user is expecting it
if( CmiGetArgFlag(argv,"+CPSamplePeriod") ||
CmiGetArgFlag(argv,"+CPSamplePeriodMs") ||
CmiGetArgFlag(argv,"+CPSchemeRandom") ||
CmiGetArgFlag(argv,"+CPExhaustiveSearch") ||
CmiGetArgFlag(argv,"+CPAlwaysUseDefaults") ||
CmiGetArgFlag(argv,"+CPSimulAnneal") ||
CmiGetArgFlag(argv,"+CPCriticalPathPrio") ||
CmiGetArgFlag(argv,"+CPBestKnown") ||
CmiGetArgFlag(argv,"+CPSteering") ||
CmiGetArgFlag(argv,"+CPMemoryAware") ||
CmiGetArgFlag(argv,"+CPSimplex") ||
CmiGetArgFlag(argv,"+CPDivideConquer") ||
CmiGetArgFlag(argv,"+CPLDBPeriod") ||
CmiGetArgFlag(argv,"+CPLDBPeriodLinear") ||
CmiGetArgFlag(argv,"+CPLDBPeriodQuadratic") ||
CmiGetArgFlag(argv,"+CPLDBPeriodOptimal") ||
CmiGetArgFlag(argv,"+CPDefaultValues") ||
CmiGetArgFlag(argv,"+CPGatherAll") ||
CmiGetArgFlag(argv,"+CPGatherMemoryUsage") ||
CmiGetArgFlag(argv,"+CPGatherUtilization") ||
CmiGetArgFlag(argv,"+CPSaveData") ||
CmiGetArgFlag(argv,"+CPNoFilterData") ||
CmiGetArgFlag(argv,"+CPLoadData") ||
CmiGetArgFlag(argv,"+CPDataFilename") )
{
CkAbort("You specified a control point command line argument, but compiled charm++ without control point support.\n");
}
#endif
}
