static void extract_args(char **argv)
{
int ms;
setspeed_gigabit();
if (CmiGetArgFlagDesc(argv,"+atm","Tune for a low-latency ATM network"))
setspeed_atm();
if (CmiGetArgFlagDesc(argv,"+eth","Tune for an ethernet network"))
setspeed_eth();
if (CmiGetArgFlagDesc(argv,"+giga","Tune for a gigabit network"))
setspeed_gigabit();
CmiGetArgIntDesc(argv,"+max_dgram_size",&Cmi_max_dgram_size,"Size of each UDP packet");
CmiGetArgIntDesc(argv,"+window_size",&Cmi_window_size,"Number of unacknowledged packets");
/* must divide for window protocol to work */
if ( (DGRAM_SEQNO_MASK+1)%Cmi_window_size != 0)
CmiAbort("Invalid window size!");
CmiGetArgIntDesc(argv,"+os_buffer_size",&Cmi_os_buffer_size, "UDP socket's SO_RCVBUF/SO_SNDBUF");
if (CmiGetArgIntDesc(argv,"+delay_retransmit",&ms, "Milliseconds to wait before retransmit"))
Cmi_delay_retransmit=0.001*ms;
if (CmiGetArgIntDesc(argv,"+ack_delay",&ms, "Milliseconds to wait before ack'ing"))
Cmi_ack_delay=0.001*ms;
extract_common_args(argv);
Cmi_dgram_max_data = Cmi_max_dgram_size - DGRAM_HEADER_SIZE;
Cmi_half_window = Cmi_window_size >> 1;
if ((Cmi_window_size * Cmi_max_dgram_size) > Cmi_os_buffer_size)
KillEveryone("Window size too big for OS buffer.");
Cmi_comm_periodic_delay=(int)(1000*Cmi_delay_retransmit);
if (Cmi_comm_periodic_delay>60) Cmi_comm_periodic_delay=60;
Cmi_comm_clock_delay=(int)(1000*Cmi_ack_delay);
if (sizeof(DgramHeader)!=DGRAM_HEADER_SIZE) {
CmiAbort("DatagramHeader in machine-dgram.c is the wrong size!\n");
}
}
TraceSimple::TraceSimple(char **argv)
{
if (CkpvAccess(traceOnPe) == 0) return;
// Process runtime arguments intended for the module
CmiGetArgIntDesc(argv,"+SimplePar0", &par0, "Fake integer parameter 0");
CmiGetArgDoubleDesc(argv,"+SimplePar1", &par1, "Fake double parameter 1");
}
void CmiInitCPUAffinity(char **argv)
{
char *pemap = NULL;
char *pemapfile = NULL;
char *commap = NULL;
int excludecore = -1;
int affinity_flag = CmiGetArgFlagDesc(argv,"+setcpuaffinity",
"set cpu affinity");
while (CmiGetArgIntDesc(argv,"+excludecore",&excludecore, "avoid core when setting cpuaffinity"));
CmiGetArgStringDesc(argv, "+pemap", &pemap, "define pe to core mapping");
CmiGetArgStringDesc(argv, "+pemapfile", &pemapfile, "define pe to core mapping file");
CmiGetArgStringDesc(argv, "+commap", &commap, "define comm threads to core mapping");
if (affinity_flag && CmiMyPe()==0)
CmiPrintf("sched_setaffinity() is not supported, +setcpuaffinity disabled.\n");
}
extern "C" void initQd(char **argv)
{
CpvInitialize(QdState*, _qd);
CpvAccess(_qd) = new QdState();
if (CmiMyRank() == 0) {
#if !defined(CMK_CPV_IS_SMP) && !CMK_SHARED_VARS_UNIPROCESSOR
CpvAccessOther(_qd, 1) = new QdState(); // for i/o interrupt
#endif
}
_qdHandlerIdx = CmiRegisterHandler((CmiHandler)_qdHandler);
_qdCommHandlerIdx = CmiRegisterHandler((CmiHandler)_qdCommHandler);
if (CmiGetArgIntDesc(argv,"+qd",&_dummy_dq, "QD time in seconds")) {
if (CmiMyPe()==0)
CmiPrintf("Charm++> Fake QD using %d seconds.\n", _dummy_dq);
}
}
TraceMemory::TraceMemory(char **argv) {
usedBuffer = 0;
firstTime = 1;
traceDisabled = false;
logBufSize = DefaultBufferSize;
if (CmiGetArgIntDesc(argv,"+memlogsize",&logBufSize,
"Log buffer size (in kB)")) {
if (CkMyPe() == 0) {
CmiPrintf("Trace: logsize: %d kB\n", logBufSize);
}
}
recordStack = false;
if (CmiGetArgFlagDesc(argv,"+recordStack",
"Record stack trace for malloc")) {
recordStack = true;
}
logBufSize *= 1024;
logBuffer = (char *) ::malloc(logBufSize);
}
void calculateNodeSizeAndRank(char **argv){
sysvshmContext->nodesize=1;
MACHSTATE(3,"calculateNodeSizeAndRank start");
CmiGetArgIntDesc(argv, "+nodesize", &(sysvshmContext->nodesize),"Number of cores in this node");
MACHSTATE1(3,"calculateNodeSizeAndRank argintdesc %d",sysvshmContext->nodesize);
sysvshmContext->noderank = _Cmi_mynode % (sysvshmContext->nodesize);
MACHSTATE1(3,"calculateNodeSizeAndRank noderank %d",sysvshmContext->noderank);
sysvshmContext->nodestart = _Cmi_mynode -sysvshmContext->noderank;
MACHSTATE(3,"calculateNodeSizeAndRank nodestart ");
sysvshmContext->nodeend = sysvshmContext->nodestart + sysvshmContext->nodesize -1;
if(sysvshmContext->nodeend >= _Cmi_numnodes){
sysvshmContext->nodeend = _Cmi_numnodes-1;
sysvshmContext->nodesize = (sysvshmContext->nodeend - sysvshmContext->nodestart) +1;
}
MACHSTATE3(3,"calculateNodeSizeAndRank nodestart %d nodesize %d noderank %d",sysvshmContext->nodestart,sysvshmContext->nodesize,sysvshmContext->noderank);
}
void calculateNodeSizeAndRank(char **argv){
pxshmContext->nodesize=1;
MACHSTATE(3,"calculateNodeSizeAndRank start");
//CmiGetArgIntDesc(argv, "+nodesize", &(pxshmContext->nodesize),"Number of cores in this node (for non-smp case).Used by the shared memory communication layer");
CmiGetArgIntDesc(argv, "+nodesize", &(pxshmContext->nodesize),"Number of cores in this node");
MACHSTATE1(3,"calculateNodeSizeAndRank argintdesc %d",pxshmContext->nodesize);
pxshmContext->noderank = _Cmi_mynode % (pxshmContext->nodesize);
MACHSTATE1(3,"calculateNodeSizeAndRank noderank %d",pxshmContext->noderank);
pxshmContext->nodestart = _Cmi_mynode -pxshmContext->noderank;
MACHSTATE(3,"calculateNodeSizeAndRank nodestart ");
pxshmContext->nodeend = pxshmContext->nodestart + pxshmContext->nodesize -1;
if(pxshmContext->nodeend >= _Cmi_numnodes){
pxshmContext->nodeend = _Cmi_numnodes-1;
pxshmContext->nodesize = (pxshmContext->nodeend - pxshmContext->nodestart) +1;
}
MACHSTATE3(3,"calculateNodeSizeAndRank nodestart %d nodesize %d noderank %d",pxshmContext->nodestart,pxshmContext->nodesize,pxshmContext->noderank);
}
void POSEreadCmdLine()
{
char **argv = CkGetArgv();
CmiArgGroup("Charm++","POSE");
pose_config.stats=CmiGetArgFlagDesc(argv, "+stats_pose",
"Gather timing information and other statistics");
/* semantic meaning for these still to be determined
CmiGetArgIntDesc(argv, "+start_proj_pose",&pose_config.start_proj,
"GVT to initiate projections tracing");
CmiGetArgIntDesc(argv, "+end_proj_pose",&pose_config.end_proj,
"GVT to end projections tracing");
*/
pose_config.trace=CmiGetArgFlagDesc(argv, "+trace_pose",
"Traces key POSE operations like Forward Execution, Rollback, Cancellation, Fossil Collection, etc. via user events for display in projections");
/* DOP command-line parameter truth table:
|---- Input ---| |------------ Output --------------|
dop dopSkipCalcs DOP logs written DOP calcs performed
--- ------------ ---------------- -------------------
F F No No
F T Yes No
T F Yes Yes
T T Yes No
*/
pose_config.dop=CmiGetArgFlagDesc(argv, "+dop_pose",
"Critical path analysis by measuring degree of parallelism");
pose_config.dopSkipCalcs=CmiGetArgFlagDesc(argv, "+dop_pose_skip_calcs",
"Records degree of parallelism logs but doesn't perform end-of-simulation calculations");
if (pose_config.dopSkipCalcs) {
pose_config.dop = true;
}
CmiGetArgIntDesc(argv, "+memman_pose", &pose_config.max_usage , "Coarse memory management: Restricts forward execution of objects with over <max_usage>/<checkpoint store_rate> checkpoints; default to 10");
/*
pose_config.msg_pool=CmiGetArgFlagDesc(argv, "+pose_msgpool", "Store and reuse pools of messages under a certain size default 1000");
CmiGetArgIntDesc(argv, "+msgpoolsize_pose", &pose_config.msg_pool_size , "Store and reuse pools of messages under a certain size default 1000");
CmiGetArgIntDesc(argv, "+msgpoolmax_pose", &pose_config.max_pool_msg_size , "Store and reuse pools of messages under a certain size");
char *strat;
CmiGetArgStringDesc(argv, "+commlib_strat_pose", &strat , "Use commlib with strat in {stream|mesh|prio}");
if(strcmp("stream",strat)==0)
pose_config.commlib_strat=stream;
if(strcmp("mesh",strat)==0)
pose_config.commlib_strat=mesh;
if(strcmp("prio",strat)==0)
pose_config.commlib_strat=prio;
CmiGetArgIntDesc(argv, "+commlib_timeout-pose", &pose_config.commlib_timeout , "Use commlib with timeout N; default 1ms");
CmiGetArgIntDesc(argv, "+commlib_maxmsg_pose", &pose_config.commlib_maxmsg , "Use commlib with max msg N; default 5");
*/
pose_config.lb_on=CmiGetArgFlagDesc(argv, "+lb_on_pose", "Use load balancing");
CmiGetArgIntDesc(argv, "+lb_skip_pose", &pose_config.lb_skip , "Load balancing skip N; default 51");
CmiGetArgIntDesc(argv, "+lb_threshold_pose", &pose_config.lb_threshold , "Load balancing threshold N; default 4000");
CmiGetArgIntDesc(argv, "+lb_diff_pose", &pose_config.lb_diff , "Load balancing min diff between min and max load PEs; default 2000");
CmiGetArgIntDesc(argv, "+checkpoint_rate_pose", &pose_config.store_rate , "Sets checkpoint to 1 for every <rate> events. Default to 1. ");
CmiGetArgIntDesc(argv, "+checkpoint_gvt_pose", &pose_config.checkpoint_gvt_interval,
"Checkpoint approximately every <gvt #> of GVT ticks; default = 0 = no checkpointing; overrides +checkpoint_time_pose");
if (pose_config.checkpoint_gvt_interval < 0) {
CmiAbort("+checkpoint_gvt_pose value must be >= 0; 0 = no checkpointing\n");
}
CmiGetArgIntDesc(argv, "+checkpoint_time_pose", &pose_config.checkpoint_time_interval,
"Checkpoint approximately every <time> seconds; default = 0 = no checkpointing; overridden by checkpoint_gvt_pose");
if (pose_config.checkpoint_time_interval < 0) {
CmiAbort("+checkpoint_time_pose value must be >= 0; 0 = no checkpointing\n");
}
if ((pose_config.checkpoint_gvt_interval > 0) && (pose_config.checkpoint_time_interval > 0)) {
CmiPrintf("WARNING: checkpoint GVT and time values both set; ignoring time value\n");
pose_config.checkpoint_time_interval = 0;
}
/* load balancing */
CmiGetArgIntDesc(argv, "+lb_gvt_pose", &pose_config.lb_gvt_interval,
"Load balancing approximately every <gvt #> of GVT ticks; default = 0 = no lb");
if (pose_config.lb_gvt_interval < 0) {
CmiAbort("+lb_gvt_pose value must be >= 0; 0 = no load balancing\n");
}
/* max_iteration seems to be defunct */
// CmiGetArgIntDesc(argv, "+FEmax_pose", &pose_config.max_iter , "Sets max events executed in single forward execution step. Default to 100.");
CmiGetArgIntDesc(argv, "+leash_specwindow_pose", &pose_config.spec_window , "Sets speculative window behavior.");
CmiGetArgIntDesc(argv, "+leash_min_pose", &pose_config.min_leash , "Sets speculative window behavior minimum leash. Default 10.");
CmiGetArgIntDesc(argv, "+leash_max_pose", &pose_config.max_leash , "Sets speculative window behavior maximum leash. Default 100.");
CmiGetArgIntDesc(argv, "+leash_flex_pose", &pose_config.max_leash , "Sets speculative window behavior leash flex. Default 10.");
if(pose_config.deterministic= CmiGetArgFlagDesc(argv, "+deterministic_pose", "sorts events of same timestamp by event id for repeatable behavior "))
{
CkPrintf("WARNING: deterministic_pose: enter at your own risk, though this feature is hopefully not broken anymore\n");
}
}
static inline void _parseCommandLineOpts(char **argv)
{
if (CmiGetArgFlagDesc(argv,"+cs", "Print extensive statistics at shutdown"))
_STATS_ON(_printCS);
if (CmiGetArgFlagDesc(argv,"+ss", "Print summary statistics at shutdown"))
_STATS_ON(_printSS);
if (CmiGetArgFlagDesc(argv,"+fifo", "Default to FIFO queuing"))
_defaultQueueing = CK_QUEUEING_FIFO;
if (CmiGetArgFlagDesc(argv,"+lifo", "Default to LIFO queuing"))
_defaultQueueing = CK_QUEUEING_LIFO;
if (CmiGetArgFlagDesc(argv,"+ififo", "Default to integer-prioritized FIFO queuing"))
_defaultQueueing = CK_QUEUEING_IFIFO;
if (CmiGetArgFlagDesc(argv,"+ilifo", "Default to integer-prioritized LIFO queuing"))
_defaultQueueing = CK_QUEUEING_ILIFO;
if (CmiGetArgFlagDesc(argv,"+bfifo", "Default to bitvector-prioritized FIFO queuing"))
_defaultQueueing = CK_QUEUEING_BFIFO;
if (CmiGetArgFlagDesc(argv,"+blifo", "Default to bitvector-prioritized LIFO queuing"))
_defaultQueueing = CK_QUEUEING_BLIFO;
if (CmiGetArgFlagDesc(argv,"+objq", "Default to use object queue for every obejct"))
{
#if CMK_OBJECT_QUEUE_AVAILABLE
_defaultObjectQ = 1;
if (CkMyPe()==0)
CmiPrintf("Charm++> Create object queue for every Charm object.\n");
#else
CmiAbort("Charm++> Object queue not enabled, recompile Charm++ with CMK_OBJECT_QUEUE_AVAILABLE defined to 1.");
#endif
}
if(CmiGetArgString(argv,"+restart",&_restartDir))
faultFunc = CkRestartMain;
#if __FAULT__
if (CmiGetArgIntDesc(argv,"+restartaftercrash",&CpvAccess(_curRestartPhase),"restarting this processor after a crash")){
# if CMK_MEM_CHECKPOINT
faultFunc = CkMemRestart;
# endif
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
faultFunc = CkMlogRestart;
#endif
CmiPrintf("[%d] Restarting after crash \n",CmiMyPe());
}
#if CMK_MESSAGE_LOGGING
// reading +ftc_disk flag
if (CmiGetArgFlagDesc(argv, "+ftc_disk", "Disk Checkpointing")) {
diskCkptFlag = 1;
}
#endif
// reading the killFile
if(CmiGetArgStringDesc(argv,"+killFile", &killFile,"Generates SIGKILL on specified processors")){
if(faultFunc == NULL){
//do not read the killfile if this is a restarting processor
killFlag = 1;
if(CmiMyPe() == 0){
printf("[%d] killFlag set to 1 for file %s\n",CkMyPe(),killFile);
}
}
}
#endif
// shut down program in ring fashion to allow projections output w/o IO error
if (CmiGetArgIntDesc(argv,"+ringexit",&_ringtoken, "Program exits in a ring fashion"))
{
_ringexit = 1;
if (CkMyPe()==0)
CkPrintf("Charm++> Program shutdown in token ring (%d).\n", _ringtoken);
if (_ringtoken > CkNumPes()) _ringtoken = CkNumPes();
}
/*
FAULT_EVAC
if the argument +raiseevac is present then cause faults
*/
if(CmiGetArgStringDesc(argv,"+raiseevac", &_raiseEvacFile,"Generates processor evacuation on random processors")){
_raiseEvac = 1;
}
#if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_))
if(!CmiGetArgIntDesc(argv,"+teamSize",&teamSize,"Set the team size for message logging")){
teamSize = 1;
}
if(!CmiGetArgIntDesc(argv,"+chkptPeriod",&chkptPeriod,"Set the checkpoint period for the message logging fault tolerance algorithm in seconds")){
chkptPeriod = 100;
}
if(CmiGetArgIntDesc(argv,"+fastRecovery", ¶llelRecovery, "Parallel recovery with message logging protocol")){
fastRecovery = true;
}
#endif
/* Anytime migration flag */
_isAnytimeMigration = true;
if (CmiGetArgFlagDesc(argv,"+noAnytimeMigration","The program does not require support for anytime migration")) {
_isAnytimeMigration = false;
}
_isNotifyChildInRed = true;
if (CmiGetArgFlagDesc(argv,"+noNotifyChildInReduction","The program has at least one element per processor for each charm array created")) {
_isNotifyChildInRed = false;
}
_isStaticInsertion = false;
if (CmiGetArgFlagDesc(argv,"+staticInsertion","Array elements are only inserted at construction")) {
_isStaticInsertion = true;
}
useNodeBlkMapping = false;
if (CmiGetArgFlagDesc(argv,"+useNodeBlkMapping","Array elements are block-mapped in SMP-node level")) {
useNodeBlkMapping = true;
}
#if ! CMK_WITH_CONTROLPOINT
// Display a warning if charm++ wasn't compiled with control point support but user is expecting it
if( CmiGetArgFlag(argv,"+CPSamplePeriod") ||
CmiGetArgFlag(argv,"+CPSamplePeriodMs") ||
CmiGetArgFlag(argv,"+CPSchemeRandom") ||
CmiGetArgFlag(argv,"+CPExhaustiveSearch") ||
CmiGetArgFlag(argv,"+CPAlwaysUseDefaults") ||
CmiGetArgFlag(argv,"+CPSimulAnneal") ||
CmiGetArgFlag(argv,"+CPCriticalPathPrio") ||
CmiGetArgFlag(argv,"+CPBestKnown") ||
CmiGetArgFlag(argv,"+CPSteering") ||
CmiGetArgFlag(argv,"+CPMemoryAware") ||
CmiGetArgFlag(argv,"+CPSimplex") ||
CmiGetArgFlag(argv,"+CPDivideConquer") ||
CmiGetArgFlag(argv,"+CPLDBPeriod") ||
CmiGetArgFlag(argv,"+CPLDBPeriodLinear") ||
CmiGetArgFlag(argv,"+CPLDBPeriodQuadratic") ||
CmiGetArgFlag(argv,"+CPLDBPeriodOptimal") ||
CmiGetArgFlag(argv,"+CPDefaultValues") ||
CmiGetArgFlag(argv,"+CPGatherAll") ||
CmiGetArgFlag(argv,"+CPGatherMemoryUsage") ||
CmiGetArgFlag(argv,"+CPGatherUtilization") ||
CmiGetArgFlag(argv,"+CPSaveData") ||
CmiGetArgFlag(argv,"+CPNoFilterData") ||
CmiGetArgFlag(argv,"+CPLoadData") ||
CmiGetArgFlag(argv,"+CPDataFilename") )
{
CkAbort("You specified a control point command line argument, but compiled charm++ without control point support.\n");
}
#endif
}
void CcsInit(char **argv)
{
CpvInitialize(CkHashtable_c, ccsTab);
CpvAccess(ccsTab) = CkCreateHashtable_string(sizeof(CcsHandlerRec),5);
CpvInitialize(CcsImplHeader *, ccsReq);
CpvAccess(ccsReq) = NULL;
_ccsHandlerIdx = CmiRegisterHandler((CmiHandler)req_fw_handler);
#if CMK_BIGSIM_CHARM
CpvInitialize(int, _bgCcsHandlerIdx);
CpvAccess(_bgCcsHandlerIdx) = 0;
CpvInitialize(int, _bgCcsAck);
CpvAccess(_bgCcsAck) = 0;
#endif
CpvInitialize(int, cmiArgDebugFlag);
CpvInitialize(char *, displayArgument);
CpvInitialize(int, cpdSuspendStartup);
CpvAccess(cmiArgDebugFlag) = 0;
CpvAccess(displayArgument) = NULL;
CpvAccess(cpdSuspendStartup) = 0;
CcsBuiltinsInit(argv);
rep_fw_handler_idx = CmiRegisterHandler((CmiHandler)rep_fw_handler);
#if NODE_0_IS_CONVHOST
#if ! CMK_CMIPRINTF_IS_A_BUILTIN
print_fw_handler_idx = CmiRegisterHandler((CmiHandler)print_fw_handler);
#endif
{
int ccs_serverPort=0;
char *ccs_serverAuth=NULL;
if (CmiGetArgFlagDesc(argv,"++server", "Create a CCS server port") |
CmiGetArgIntDesc(argv,"++server-port",&ccs_serverPort, "Listen on this TCP/IP port number") |
CmiGetArgStringDesc(argv,"++server-auth",&ccs_serverAuth, "Use this CCS authentication file"))
if (CmiMyPe()==0)
{/*Create and occasionally poll on a CCS server port*/
CcsServer_new(NULL,&ccs_serverPort,ccs_serverAuth);
CcdCallOnConditionKeep(CcdPERIODIC,(CcdVoidFn)CcsServerCheck,NULL);
}
}
#endif
/* if in parallel debug mode i.e ++cpd, freeze */
if (CmiGetArgFlagDesc(argv, "+cpd", "Used *only* in conjunction with parallel debugger"))
{
CpvAccess(cmiArgDebugFlag) = 1;
if (CmiGetArgStringDesc(argv, "+DebugDisplay",&(CpvAccess(displayArgument)), "X display for gdb used only in cpd mode"))
{
if (CpvAccess(displayArgument) == NULL)
CmiPrintf("WARNING> NULL parameter for +DebugDisplay\n***");
}
else if (CmiMyPe() == 0)
{
/* only one processor prints the warning */
CmiPrintf("WARNING> x term for gdb needs to be specified as +DebugDisplay by debugger\n***\n");
}
if (CmiGetArgFlagDesc(argv, "+DebugSuspend", "Suspend execution at beginning of program")) {
CpvAccess(cpdSuspendStartup) = 1;
}
}
CcsReleaseMessages();
}
// called from init.C
void _loadbalancerInit()
{
CkpvInitialize(int, lbdatabaseInited);
CkpvAccess(lbdatabaseInited) = 0;
CkpvInitialize(int, numLoadBalancers);
CkpvAccess(numLoadBalancers) = 0;
CkpvInitialize(int, hasNullLB);
CkpvAccess(hasNullLB) = 0;
char **argv = CkGetArgv();
char *balancer = NULL;
CmiArgGroup("Charm++","Load Balancer");
while (CmiGetArgStringDesc(argv, "+balancer", &balancer, "Use this load balancer")) {
if (CkMyRank() == 0)
lbRegistry.addRuntimeBalancer(balancer); /* lbRegistry is a static */
}
// set up init value for LBPeriod time in seconds
// it can also be set by calling LDSetLBPeriod()
CmiGetArgDoubleDesc(argv,"+LBPeriod", &_lb_args.lbperiod(),"the minimum time period in seconds allowed for two consecutive automatic load balancing");
_lb_args.loop() = CmiGetArgFlagDesc(argv, "+LBLoop", "Use multiple load balancing strategies in loop");
// now called in cldb.c: CldModuleGeneralInit()
// registerLBTopos();
CmiGetArgStringDesc(argv, "+LBTopo", &_lbtopo, "define load balancing topology");
//Read the K parameter for RefineKLB
CmiGetArgIntDesc(argv, "+LBNumMoves", &_lb_args.percentMovesAllowed() , "Percentage of chares to be moved (used by RefineKLB) [0-100]");
/**************** FUTURE PREDICTOR ****************/
_lb_predict = CmiGetArgFlagDesc(argv, "+LBPredictor", "Turn on LB future predictor");
CmiGetArgIntDesc(argv, "+LBPredictorDelay", &_lb_predict_delay, "Number of balance steps before learning a model");
CmiGetArgIntDesc(argv, "+LBPredictorWindow", &_lb_predict_window, "Number of steps to use to learn a model");
if (_lb_predict_window < _lb_predict_delay) {
CmiPrintf("LB> [%d] Argument LBPredictorWindow (%d) less than LBPredictorDelay (%d) , fixing\n", CkMyPe(), _lb_predict_window, _lb_predict_delay);
_lb_predict_delay = _lb_predict_window;
}
/******************* SIMULATION *******************/
// get the step number at which to dump the LB database
CmiGetArgIntDesc(argv, "+LBVersion", &_lb_args.lbversion(), "LB database file version number");
CmiGetArgIntDesc(argv, "+LBCentPE", &_lb_args.central_pe(), "CentralLB processor");
int _lb_dump_activated = 0;
if (CmiGetArgIntDesc(argv, "+LBDump", &LBSimulation::dumpStep, "Dump the LB state from this step"))
_lb_dump_activated = 1;
if (_lb_dump_activated && LBSimulation::dumpStep < 0) {
CmiPrintf("LB> Argument LBDump (%d) negative, setting to 0\n",LBSimulation::dumpStep);
LBSimulation::dumpStep = 0;
}
CmiGetArgIntDesc(argv, "+LBDumpSteps", &LBSimulation::dumpStepSize, "Dump the LB state for this amount of steps");
if (LBSimulation::dumpStepSize <= 0) {
CmiPrintf("LB> Argument LBDumpSteps (%d) too small, setting to 1\n",LBSimulation::dumpStepSize);
LBSimulation::dumpStepSize = 1;
}
CmiGetArgStringDesc(argv, "+LBDumpFile", &LBSimulation::dumpFile, "Set the LB state file name");
// get the simulation flag and number. Now the flag can also be avoided by the presence of the number
LBSimulation::doSimulation = CmiGetArgIntDesc(argv, "+LBSim", &LBSimulation::simStep, "Read LB state from LBDumpFile since this step");
// check for stupid LBSim parameter
if (LBSimulation::doSimulation && LBSimulation::simStep < 0) {
CmiPrintf("LB> Argument LBSim (%d) invalid, should be >= 0\n");
CkExit();
return;
}
CmiGetArgIntDesc(argv, "+LBSimSteps", &LBSimulation::simStepSize, "Read LB state for this number of steps");
if (LBSimulation::simStepSize <= 0) {
CmiPrintf("LB> Argument LBSimSteps (%d) too small, setting to 1\n",LBSimulation::simStepSize);
LBSimulation::simStepSize = 1;
}
LBSimulation::simProcs = 0;
CmiGetArgIntDesc(argv, "+LBSimProcs", &LBSimulation::simProcs, "Number of target processors.");
LBSimulation::showDecisionsOnly =
CmiGetArgFlagDesc(argv, "+LBShowDecisions",
"Write to File: Load Balancing Object to Processor Map decisions during LB Simulation");
// force a global barrier after migration done
_lb_args.syncResume() = CmiGetArgFlagDesc(argv, "+LBSyncResume",
"LB performs a barrier after migration is finished");
// both +LBDebug and +LBDebug level should work
if (!CmiGetArgIntDesc(argv, "+LBDebug", &_lb_args.debug(),
"Turn on LB debugging printouts"))
_lb_args.debug() = CmiGetArgFlagDesc(argv, "+LBDebug",
"Turn on LB debugging printouts");
// getting the size of the team with +teamSize
if (!CmiGetArgIntDesc(argv, "+teamSize", &_lb_args.teamSize(),
"Team size"))
_lb_args.teamSize() = 1;
// ask to print summary/quality of load balancer
_lb_args.printSummary() = CmiGetArgFlagDesc(argv, "+LBPrintSummary",
"Print load balancing result summary");
// to ignore baclground load
_lb_args.ignoreBgLoad() = CmiGetArgFlagDesc(argv, "+LBNoBackground",
"Load balancer ignores the background load.");
#ifdef __BIGSIM__
_lb_args.ignoreBgLoad() = 1;
#endif
_lb_args.migObjOnly() = CmiGetArgFlagDesc(argv, "+LBObjOnly",
"Only load balancing migratable objects, ignoring all others.");
if (_lb_args.migObjOnly()) _lb_args.ignoreBgLoad() = 1;
// assume all CPUs are identical
_lb_args.testPeSpeed() = CmiGetArgFlagDesc(argv, "+LBTestPESpeed",
"Load balancer test all CPUs speed.");
_lb_args.samePeSpeed() = CmiGetArgFlagDesc(argv, "+LBSameCpus",
"Load balancer assumes all CPUs are of same speed.");
if (!_lb_args.testPeSpeed()) _lb_args.samePeSpeed() = 1;
_lb_args.useCpuTime() = CmiGetArgFlagDesc(argv, "+LBUseCpuTime",
"Load balancer uses CPU time instead of wallclock time.");
// turn instrumentation off at startup
_lb_args.statsOn() = !CmiGetArgFlagDesc(argv, "+LBOff",
"Turn load balancer instrumentation off");
// turn instrumentation of communicatin off at startup
_lb_args.traceComm() = !CmiGetArgFlagDesc(argv, "+LBCommOff",
"Turn load balancer instrumentation of communication off");
// set alpha and beeta
_lb_args.alpha() = PER_MESSAGE_SEND_OVERHEAD_DEFAULT;
_lb_args.beeta() = PER_BYTE_SEND_OVERHEAD_DEFAULT;
CmiGetArgDoubleDesc(argv,"+LBAlpha", &_lb_args.alpha(),
"per message send overhead");
CmiGetArgDoubleDesc(argv,"+LBBeta", &_lb_args.beeta(),
"per byte send overhead");
if (CkMyPe() == 0) {
if (_lb_args.debug()) {
CmiPrintf("CharmLB> Verbose level %d, load balancing period: %g seconds\n", _lb_args.debug(), _lb_args.lbperiod());
}
if (_lb_args.debug() > 1) {
CmiPrintf("CharmLB> Topology %s alpha: %es beta: %es.\n", _lbtopo, _lb_args.alpha(), _lb_args.beeta());
}
if (_lb_args.printSummary())
CmiPrintf("CharmLB> Load balancer print summary of load balancing result.\n");
if (_lb_args.ignoreBgLoad())
CmiPrintf("CharmLB> Load balancer ignores processor background load.\n");
if (_lb_args.samePeSpeed())
CmiPrintf("CharmLB> Load balancer assumes all CPUs are same.\n");
if (_lb_args.useCpuTime())
CmiPrintf("CharmLB> Load balancer uses CPU time instead of wallclock time.\n");
if (LBSimulation::doSimulation)
CmiPrintf("CharmLB> Load balancer running in simulation mode on file '%s' version %d.\n", LBSimulation::dumpFile, _lb_args.lbversion());
if (_lb_args.statsOn()==0)
CkPrintf("CharmLB> Load balancing instrumentation is off.\n");
if (_lb_args.traceComm()==0)
CkPrintf("CharmLB> Load balancing instrumentation for communication is off.\n");
if (_lb_args.migObjOnly())
CkPrintf("LB> Load balancing strategy ignores non-migratable objects.\n");
}
}
void CmiInitCPUAffinity(char **argv)
{
static skt_ip_t myip;
int ret, i, exclude;
hostnameMsg *msg;
char *pemap = NULL;
char *commap = NULL;
char *pemapfile = NULL;
int show_affinity_flag;
int affinity_flag = CmiGetArgFlagDesc(argv,"+setcpuaffinity",
"set cpu affinity");
while (CmiGetArgIntDesc(argv,"+excludecore", &exclude, "avoid core when setting cpuaffinity")) {
if (CmiMyRank() == 0) add_exclude(exclude);
affinity_flag = 1;
}
if (CmiGetArgStringDesc(argv, "+pemapfile", &pemapfile, "define pe to core mapping file")) {
FILE *fp;
char buf[128];
pemap = (char*)malloc(1024);
fp = fopen(pemapfile, "r");
if (fp == NULL) CmiAbort("pemapfile does not exist");
while (!feof(fp)) {
if (fgets(buf, 128, fp)) {
if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
strcat(pemap, buf);
}
}
fclose(fp);
if (CmiMyPe()==0) CmiPrintf("Charm++> read from pemap file '%s': %s\n", pemapfile, pemap);
}
CmiGetArgStringDesc(argv, "+pemap", &pemap, "define pe to core mapping");
if (pemap!=NULL && excludecount>0)
CmiAbort("Charm++> +pemap can not be used with +excludecore.\n");
CmiGetArgStringDesc(argv, "+commap", &commap, "define comm threads to core mapping");
if (pemap!=NULL || commap!=NULL) affinity_flag = 1;
show_affinity_flag = CmiGetArgFlagDesc(argv,"+showcpuaffinity",
"print cpu affinity");
cpuAffinityHandlerIdx =
CmiRegisterHandler((CmiHandler)cpuAffinityHandler);
cpuAffinityRecvHandlerIdx =
CmiRegisterHandler((CmiHandler)cpuAffinityRecvHandler);
if (CmiMyRank() ==0) {
affLock = CmiCreateLock();
}
#if CMK_BLUEGENEP || CMK_BLUEGENEQ
if(affinity_flag){
affinity_flag = 0;
if(CmiMyPe()==0) CmiPrintf("Charm++> cpu affinity setting is not needed on Blue Gene, thus ignored.\n");
}
if(show_affinity_flag){
show_affinity_flag = 0;
if(CmiMyPe()==0) CmiPrintf("Charm++> printing cpu affinity is not supported on Blue Gene.\n");
}
#endif
if (!affinity_flag) {
if (show_affinity_flag) CmiPrintCPUAffinity();
return;
}
if (CmiMyPe() == 0) {
CmiPrintf("Charm++> cpu affinity enabled. \n");
if (excludecount > 0) {
CmiPrintf("Charm++> cpuaffinity excludes core: %d", excludecore[0]);
for (i=1; i<excludecount; i++) CmiPrintf(" %d", excludecore[i]);
CmiPrintf(".\n");
}
if (pemap!=NULL)
CmiPrintf("Charm++> cpuaffinity PE-core map : %s\n", pemap);
}
if (CmiMyPe() >= CmiNumPes()) { /* this is comm thread */
/* comm thread either can float around, or pin down to the last rank.
however it seems to be reportedly slower if it is floating */
CmiNodeAllBarrier();
if (commap != NULL) {
int mycore = search_pemap(commap, CmiMyPeGlobal()-CmiNumPesGlobal());
if(CmiMyPe()-CmiNumPes()==0) printf("Charm++> set comm %d on node %d to core #%d\n", CmiMyPe()-CmiNumPes(), CmiMyNode(), mycore);
if (-1 == CmiSetCPUAffinity(mycore))
CmiAbort("set_cpu_affinity abort!");
CmiNodeAllBarrier();
if (show_affinity_flag) CmiPrintCPUAffinity();
return; /* comm thread return */
}
else {
/* if (CmiSetCPUAffinity(CmiNumCores()-1) == -1) CmiAbort("set_cpu_affinity abort!"); */
#if !CMK_CRAYXT && !CMK_CRAYXE && !CMK_CRAYXC && !CMK_BLUEGENEQ
if (pemap == NULL) {
#if CMK_MACHINE_PROGRESS_DEFINED
while (affinity_doneflag < CmiMyNodeSize()) CmiNetworkProgress();
#else
#if CMK_SMP
#error "Machine progress call needs to be implemented for cpu affinity!"
#endif
#endif
}
#endif
#if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
/* if both pemap and commmap are NULL, will compute one */
if (pemap != NULL)
#endif
{
CmiNodeAllBarrier();
if (show_affinity_flag) CmiPrintCPUAffinity();
return; /* comm thread return */
}
}
}
if (pemap != NULL && CmiMyPe()<CmiNumPes()) { /* work thread */
int mycore = search_pemap(pemap, CmiMyPeGlobal());
if(show_affinity_flag) CmiPrintf("Charm++> set PE %d on node %d to core #%d\n", CmiMyPe(), CmiMyNode(), mycore);
if (mycore >= CmiNumCores()) {
CmiPrintf("Error> Invalid core number %d, only have %d cores (0-%d) on the node. \n", mycore, CmiNumCores(), CmiNumCores()-1);
CmiAbort("Invalid core number");
}
if (CmiSetCPUAffinity(mycore) == -1) CmiAbort("set_cpu_affinity abort!");
CmiNodeAllBarrier();
CmiNodeAllBarrier();
/* if (show_affinity_flag) CmiPrintCPUAffinity(); */
return;
}
#if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
{
int numCores = CmiNumCores();
int myid = getXTNodeID(CmiMyNodeGlobal(), CmiNumNodesGlobal());
int myrank;
int pe, mype = CmiMyPeGlobal();
int node = CmiMyNodeGlobal();
int nnodes = 0;
#if CMK_SMP
if (CmiMyPe() >= CmiNumPes()) { /* this is comm thread */
int node = CmiMyPe() - CmiNumPes();
mype = CmiGetPeGlobal(CmiNodeFirst(node) + CmiMyNodeSize() - 1, CmiMyPartition()); /* last pe on SMP node */
node = CmiGetNodeGlobal(node, CmiMyPartition());
}
#endif
pe = mype - 1;
while (pe >= 0) {
int n = CmiNodeOf(pe);
if (n != node) { nnodes++; node = n; }
if (getXTNodeID(n, CmiNumNodesGlobal()) != myid) break;
pe --;
}
CmiAssert(numCores > 0);
myrank = (mype - pe - 1 + nnodes)%numCores;
#if CMK_SMP
if (CmiMyPe() >= CmiNumPes())
myrank = (myrank + 1)%numCores;
#endif
if (-1 != CmiSetCPUAffinity(myrank)) {
DEBUGP(("Processor %d is bound to core #%d on node #%d\n", CmiMyPe(), myrank, mynode));
}
else{
CmiPrintf("Processor %d set affinity failed!\n", CmiMyPe());
CmiAbort("set cpu affinity abort!\n");
}
}
if (CmiMyPe() < CmiNumPes())
CmiNodeAllBarrier();
CmiNodeAllBarrier();
#else
/* get my ip address */
if (CmiMyRank() == 0)
{
#if CMK_HAS_GETHOSTNAME
myip = skt_my_ip(); /* not thread safe, so only calls on rank 0 */
#else
CmiAbort("Can not get unique name for the compute nodes. \n");
#endif
}
CmiNodeAllBarrier();
/* prepare a msg to send */
msg = (hostnameMsg *)CmiAlloc(sizeof(hostnameMsg));
CmiSetHandler((char *)msg, cpuAffinityHandlerIdx);
msg->pe = CmiMyPe();
msg->ip = myip;
msg->ncores = CmiNumCores();
DEBUGP(("PE %d's node has %d number of cores. \n", CmiMyPe(), msg->ncores));
msg->rank = 0;
CmiSyncSendAndFree(0, sizeof(hostnameMsg), (void *)msg);
if (CmiMyPe() == 0) {
int i;
hostTable = CmmNew();
rankmsg = (rankMsg *)CmiAlloc(sizeof(rankMsg)+CmiNumPes()*sizeof(int)*2);
CmiSetHandler((char *)rankmsg, cpuAffinityRecvHandlerIdx);
rankmsg->ranks = (int *)((char*)rankmsg + sizeof(rankMsg));
rankmsg->nodes = (int *)((char*)rankmsg + sizeof(rankMsg) + CmiNumPes()*sizeof(int));
for (i=0; i<CmiNumPes(); i++) {
rankmsg->ranks[i] = 0;
rankmsg->nodes[i] = -1;
}
for (i=0; i<CmiNumPes(); i++) CmiDeliverSpecificMsg(cpuAffinityHandlerIdx);
}
/* receive broadcast from PE 0 */
CmiDeliverSpecificMsg(cpuAffinityRecvHandlerIdx);
CmiLock(affLock);
affinity_doneflag++;
CmiUnlock(affLock);
CmiNodeAllBarrier();
#endif
if (show_affinity_flag) CmiPrintCPUAffinity();
}
