void craynid_init() { static init_done = 0; if (!init_done) { cray_lock = CmiCreateLock(); cray_lock2 = CmiCreateLock(); init_done = 1; } }
void CmiNodeStateInit(CmiNodeState *nodeState) { MACHSTATE1(4,"NodeStateInit %p", nodeState) #if CMK_IMMEDIATE_MSG nodeState->immSendLock = CmiCreateLock(); nodeState->immRecvLock = CmiCreateLock(); nodeState->immQ = CMIQueueCreate(); nodeState->delayedImmQ = CMIQueueCreate(); #endif #if CMK_NODE_QUEUE_AVAILABLE nodeState->CmiNodeRecvLock = CmiCreateLock(); nodeState->NodeRecv = CMIQueueCreate(); #endif MACHSTATE(4,"NodeStateInit done") }
void bgq_topo_init() { static int init_done = 0; if (!init_done) { bgq_lock = CmiCreateLock(); init_done = 1; } }
CDECL void FEM_Init(FEM_Comm_t defaultComm) { IDXL_Init(defaultComm); if (!femLock) femLock = CmiCreateLock(); if (!TCHARM_Get_global(FEM_globalID)) { FEMchunk *c=new FEMchunk(defaultComm); TCHARM_Set_global(FEM_globalID,c,pupFEM_Chunk); } }
void CkArrayReductionMgr::init() { //ARPRINT("Array ReductionMgr Constructor called %d\n",thisgroup); redNo=0; size = CkMyNodeSize(); count = 0; lockCount = CmiCreateLock(); ctorDoneFlag = 1; alreadyStarted = -1; }
void CkArrayReductionMgr::pup(PUP::er &p){ NodeGroup::pup(p); p(redNo);p(count); p|my_msgs; p|my_futureMsgs; p|attachedGroup; if(p.isUnpacking()) { size = CkMyNodeSize(); lockCount = CmiCreateLock(); } }
/** * Obtain the number of nodes, my node id, and consuming machine layer * specific arguments */ static void MachineInitForLAPI(int *argc, char ***argv, int *numNodes, int *myNodeID) { lapi_info_t info; char **largv = *argv; memset(&info,0,sizeof(info)); /* Register error handler (redundant?) -- added by Chao Mei*/ info.err_hndlr = (LAPI_err_hndlr *)lapi_err_hndlr; /* Indicates the number of completion handler threads to create */ /* The number of completion hndlr thds will affect the atomic PCQueue operations!! */ /* NOTE: num_compl_hndlr_thr is obsolete now! --Chao Mei */ /* info.num_compl_hndlr_thr = 1; */ check_lapi(LAPI_Init,(&lapiContext, &info)); /* It's a good idea to start with a fence, because packets recv'd before a LAPI_Init are just dropped. */ check_lapi(LAPI_Gfence,(lapiContext)); check_lapi(LAPI_Qenv,(lapiContext, TASK_ID, myNodeID)); check_lapi(LAPI_Qenv,(lapiContext, NUM_TASKS, numNodes)); /* Make polling as the default mode as real apps have better perf */ CsvAccess(lapiInterruptMode) = 0; if (CmiGetArgFlag(largv,"+poll")) CsvAccess(lapiInterruptMode) = 0; if (CmiGetArgFlag(largv,"+nopoll")) CsvAccess(lapiInterruptMode) = 1; check_lapi(LAPI_Senv,(lapiContext, ERROR_CHK, lapiDebugMode)); check_lapi(LAPI_Senv,(lapiContext, INTERRUPT_SET, CsvAccess(lapiInterruptMode))); if (*myNodeID == 0) { printf("Running lapi in interrupt mode: %d\n", CsvAccess(lapiInterruptMode)); printf("Running lapi with %d completion handler threads.\n", info.num_compl_hndlr_thr); } /** * Associate PumpMsgsBegin with var "lapiHeaderHandler". Then inside Xfer calls, * lapiHeaderHandler could be used to indicate the callback * instead of PumpMsgsBegin --Chao Mei */ check_lapi(LAPI_Addr_set,(lapiContext,(void *)PumpMsgsBegin,lapiHeaderHandler)); if (CmiGetArgFlag(largv,"++debug")) { /*Pause so user has a chance to start and attach debugger*/ printf("CHARMDEBUG> Processor %d has PID %d\n",*myNodeID,getpid()); if (!CmiGetArgFlag(largv,"++debug-no-pause")) sleep(30); } #if ENSURE_MSG_PAIRORDER cmplHdlrThdLock = CmiCreateLock(); #endif }
static void CmiStartThreads(char **argv) { int i,tocreate; DWORD threadID; HANDLE thr; CmiMemLock_lock=CmiCreateLock(); comm_mutex = CmiCreateLock(); barrier_mutex = CmiCreateLock(); #ifdef CMK_NO_ASM_AVAILABLE cmiMemoryLock = CmiCreateLock(); if (CmiMyNode()==0) printf("Charm++ warning> fences and atomic operations not available in native assembly\n"); #endif Cmi_state_key = TlsAlloc(); if(Cmi_state_key == 0xFFFFFFFF) PerrorExit("TlsAlloc main"); Cmi_state_vector = (CmiState)calloc(_Cmi_mynodesize+1, sizeof(struct CmiStateStruct)); for (i=0; i<_Cmi_mynodesize; i++) CmiStateInit(i+Cmi_nodestart, i, CmiGetStateN(i)); /*Create a fake state structure for the comm. thread*/ /* CmiStateInit(-1,_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); */ CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); #if CMK_MULTICORE || CMK_SMP_NO_COMMTHD if (!Cmi_commthread) tocreate = _Cmi_mynodesize-1; else #endif tocreate = _Cmi_mynodesize; for (i=1; i<=tocreate; i++) { if((thr = CreateThread(NULL, 0, call_startfn, (LPVOID)i, 0, &threadID)) == NULL) PerrorExit("CreateThread"); CloseHandle(thr); } if(TlsSetValue(Cmi_state_key, (LPVOID)Cmi_state_vector) == 0) PerrorExit("TlsSetValue"); }
void LBDatabase::initnodeFn() { int proc; int num_proc = CkNumPes(); avail_vector= new char[num_proc]; for(proc = 0; proc < num_proc; proc++) avail_vector[proc] = 1; avail_vector_lock = CmiCreateLock(); _expectedLoad = new LBRealType[num_proc]; for (proc=0; proc<num_proc; proc++) _expectedLoad[proc]=0.0; }
void LBDatabase::initnodeFn() { int proc; int num_proc = CkNumPes(); avail_vector= new char[num_proc]; for(proc = 0; proc < num_proc; proc++) avail_vector[proc] = 1; avail_vector_lock = CmiCreateLock(); _expectedLoad = new LBRealType[num_proc]; for (proc=0; proc<num_proc; proc++) _expectedLoad[proc]=0.0; _registerCommandLineOpt("+balancer"); _registerCommandLineOpt("+LBPeriod"); _registerCommandLineOpt("+LBLoop"); _registerCommandLineOpt("+LBTopo"); _registerCommandLineOpt("+LBNumMoves"); _registerCommandLineOpt("+LBPredictor"); _registerCommandLineOpt("+LBPredictorDelay"); _registerCommandLineOpt("+LBPredictorWindow"); _registerCommandLineOpt("+LBVersion"); _registerCommandLineOpt("+LBCentPE"); _registerCommandLineOpt("+LBDump"); _registerCommandLineOpt("+LBDumpSteps"); _registerCommandLineOpt("+LBDumpFile"); _registerCommandLineOpt("+LBSim"); _registerCommandLineOpt("+LBSimSteps"); _registerCommandLineOpt("+LBSimProcs"); _registerCommandLineOpt("+LBShowDecisions"); _registerCommandLineOpt("+LBSyncResume"); _registerCommandLineOpt("+LBDebug"); _registerCommandLineOpt("+teamSize"); _registerCommandLineOpt("+LBPrintSummary"); _registerCommandLineOpt("+LBNoBackground"); _registerCommandLineOpt("+LBObjOnly"); _registerCommandLineOpt("+LBTestPESpeed"); _registerCommandLineOpt("+LBSameCpus"); _registerCommandLineOpt("+LBUseCpuTime"); _registerCommandLineOpt("+LBOff"); _registerCommandLineOpt("+LBCommOff"); _registerCommandLineOpt("+MetaLB"); _registerCommandLineOpt("+LBAlpha"); _registerCommandLineOpt("+LBBeta"); }
ProxyPatch::ProxyPatch(PatchID pd) : Patch(pd), proxyMsgBufferStatus(PROXYMSGNOTBUFFERED), curProxyMsg(NULL), prevProxyMsg(NULL) { DebugM(4, "ProxyPatch(" << pd << ") at " << this << "\n"); ProxyMgr::Object()->registerProxy(patchID); numAtoms = -1; parent = -1; #ifndef NODEAWARE_PROXY_SPANNINGTREE nChild = 0; child = new int[proxySpanDim]; #endif #if CMK_PERSISTENT_COMM && USE_PERSISTENT_TREE localphs = 0; #ifdef REMOVE_PROXYRESULTMSG_EXTRACOPY int msgstart = sizeof(envelope)+sizeof(ProxyResultVarsizeMsg); #else int msgstart = sizeof(envelope)+sizeof(ProxyResultMsg); #endif localphs = CmiCreatePersistent(PatchMap::Object()->node(patchID), 30000, msgstart); ntreephs = 0; #ifdef NODEAWARE_PROXY_SPANNINGTREE treephs = NULL; #else treephs = new PersistentHandle[proxySpanDim]; #endif #endif // DMK - Atom Separation (water vs. non-water) #if NAMD_SeparateWaters != 0 numWaterAtoms = -1; #endif #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) depositLock = CmiCreateLock(); #endif }
PsiCache::PsiCache() { GWBSE *gwbse = GWBSE::get(); K = gwbse->gw_parallel.K; L = gwbse->gw_parallel.L; qindex = Q_IDX; psi_size = gwbse->gw_parallel.n_elems; pipeline_stages = gwbse->gw_parallel.pipeline_stages; received_psis = 0; received_chunks = 0; psis = new complex**[K]; for (int k = 0; k < K; k++) { psis[k] = new complex*[L]; for (int l = 0; l < L; l++) { psis[k][l] = new complex[psi_size]; } } // shifted k grid psis. Need this for qindex=0 psis_shifted = new complex**[K]; for (int k = 0; k < K; k++) { psis_shifted[k] = new complex*[L]; for (int l = 0; l < L; l++) { psis_shifted[k][l] = new complex[psi_size]; } } fs = new complex[L*psi_size*pipeline_stages]; umklapp_factor = new complex[psi_size]; // Variables for chare region registration min_row = INT_MAX; min_col = INT_MAX; max_row = INT_MIN; max_col = INT_MIN; tile_lock = CmiCreateLock(); total_time = 0.0; contribute(CkCallback(CkReductionTarget(Controller,psiCacheReady), controller_proxy)); }
void init_plan_lock() { fft_plan_lock = CmiCreateLock(); }
/** This is the main charm setup routine. It's called on all processors after Converse initialization. This routine gets passed to Converse from "main.C". The main purpose of this routine is to set up the objects and Ckpv's used during a regular Charm run. See the comment at the top of the file for overall flow. */ void _initCharm(int unused_argc, char **argv) { int inCommThread = (CmiMyRank() == CmiMyNodeSize()); DEBUGF(("[%d,%.6lf ] _initCharm started\n",CmiMyPe(),CmiWallTimer())); CkpvInitialize(size_t *, _offsets); CkpvAccess(_offsets) = new size_t[32]; CkpvInitialize(PtrQ*,_buffQ); CkpvInitialize(PtrVec*,_bocInitVec); CkpvInitialize(void*, _currentChare); CkpvInitialize(int, _currentChareType); CkpvInitialize(CkGroupID, _currentGroup); CkpvInitialize(void *, _currentNodeGroupObj); CkpvInitialize(CkGroupID, _currentGroupRednMgr); CkpvInitialize(GroupTable*, _groupTable); CkpvInitialize(GroupIDTable*, _groupIDTable); CkpvInitialize(CmiImmediateLockType, _groupTableImmLock); CkpvInitialize(bool, _destroyingNodeGroup); CkpvAccess(_destroyingNodeGroup) = false; CkpvInitialize(UInt, _numGroups); CkpvInitialize(int, _numInitsRecd); CkpvInitialize(int, _initdone); CkpvInitialize(char**, Ck_argv); CkpvAccess(Ck_argv)=argv; CkpvInitialize(MsgPool*, _msgPool); CkpvInitialize(CkCoreState *, _coreState); /* Added for evacuation-sayantan */ #ifndef __BIGSIM__ CpvInitialize(char *,_validProcessors); #endif CkpvInitialize(char ,startedEvac); CpvInitialize(int,serializer); _initChareTables(); // for checkpointable plain chares CksvInitialize(UInt, _numNodeGroups); CksvInitialize(GroupTable*, _nodeGroupTable); CksvInitialize(GroupIDTable, _nodeGroupIDTable); CksvInitialize(CmiImmediateLockType, _nodeGroupTableImmLock); CksvInitialize(CmiNodeLock, _nodeLock); CksvInitialize(PtrVec*,_nodeBocInitVec); CksvInitialize(UInt,_numInitNodeMsgs); CkpvInitialize(int,_charmEpoch); CkpvAccess(_charmEpoch)=0; CksvInitialize(int, _triggersSent); CksvAccess(_triggersSent) = 0; CkpvInitialize(_CkOutStream*, _ckout); CkpvInitialize(_CkErrStream*, _ckerr); CkpvInitialize(Stats*, _myStats); CkpvAccess(_groupIDTable) = new GroupIDTable(0); CkpvAccess(_groupTable) = new GroupTable; CkpvAccess(_groupTable)->init(); CkpvAccess(_groupTableImmLock) = CmiCreateImmediateLock(); CkpvAccess(_numGroups) = 1; // make 0 an invalid group number CkpvAccess(_buffQ) = new PtrQ(); CkpvAccess(_bocInitVec) = new PtrVec(); CkpvAccess(_currentNodeGroupObj) = NULL; if(CkMyRank()==0) { CksvAccess(_numNodeGroups) = 1; //make 0 an invalid group number CksvAccess(_numInitNodeMsgs) = 0; CksvAccess(_nodeLock) = CmiCreateLock(); CksvAccess(_nodeGroupTable) = new GroupTable(); CksvAccess(_nodeGroupTable)->init(); CksvAccess(_nodeGroupTableImmLock) = CmiCreateImmediateLock(); CksvAccess(_nodeBocInitVec) = new PtrVec(); } CkCallbackInit(); CmiNodeAllBarrier(); #if ! CMK_BIGSIM_CHARM initQd(argv); // bigsim calls it in ConverseCommonInit #endif CkpvAccess(_coreState)=new CkCoreState(); CkpvAccess(_numInitsRecd) = 0; CkpvAccess(_initdone) = 0; CkpvAccess(_ckout) = new _CkOutStream(); CkpvAccess(_ckerr) = new _CkErrStream(); _charmHandlerIdx = CkRegisterHandler((CmiHandler)_bufferHandler); _initHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler); CkNumberHandlerEx(_initHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState)); _roRestartHandlerIdx = CkRegisterHandler((CmiHandler)_roRestartHandler); _exitHandlerIdx = CkRegisterHandler((CmiHandler)_exitHandler); //added for interoperabilitY _libExitHandlerIdx = CkRegisterHandler((CmiHandler)_libExitHandler); _bocHandlerIdx = CkRegisterHandler((CmiHandler)_initHandler); CkNumberHandlerEx(_bocHandlerIdx, (CmiHandlerEx)_initHandler, CkpvAccess(_coreState)); #ifdef __BIGSIM__ if(BgNodeRank()==0) #endif _infoIdx = CldRegisterInfoFn((CldInfoFn)_infoFn); _triggerHandlerIdx = CkRegisterHandler((CmiHandler)_triggerHandler); _ckModuleInit(); CldRegisterEstimator((CldEstimator)_charmLoadEstimator); _futuresModuleInit(); // part of futures implementation is a converse module _loadbalancerInit(); _metabalancerInit(); #if CMK_MEM_CHECKPOINT init_memcheckpt(argv); #endif initCharmProjections(); #if CMK_TRACE_IN_CHARM // initialize trace module in ck traceCharmInit(argv); #endif CkpvInitialize(int, envelopeEventID); CkpvAccess(envelopeEventID) = 0; CkMessageWatcherInit(argv,CkpvAccess(_coreState)); /** The rank-0 processor of each node calls the translator-generated "_register" routines. _register routines call the charm.h "CkRegister*" routines, which record function pointers and class information for all Charm entities, like Chares, Arrays, and readonlies. There's one _register routine generated for each .ci file. _register routines *must* be called in the same order on every node, and *must not* be called by multiple threads simultaniously. */ #ifdef __BIGSIM__ if(BgNodeRank()==0) #else if(CkMyRank()==0) #endif { SDAG::registerPUPables(); CmiArgGroup("Charm++",NULL); _parseCommandLineOpts(argv); _registerInit(); CkRegisterMsg("System", 0, 0, CkFreeMsg, sizeof(int)); CkRegisterChareInCharm(CkRegisterChare("null", 0, TypeChare)); CkIndex_Chare::__idx=CkRegisterChare("Chare", sizeof(Chare), TypeChare); CkRegisterChareInCharm(CkIndex_Chare::__idx); CkIndex_Group::__idx=CkRegisterChare("Group", sizeof(Group), TypeGroup); CkRegisterChareInCharm(CkIndex_Group::__idx); CkRegisterEp("null", (CkCallFnPtr)_nullFn, 0, 0, 0+CK_EP_INTRINSIC); /** These _register calls are for the built-in Charm .ci files, like arrays and load balancing. If you add a .ci file to charm, you'll have to add a call to the _register routine here, or make your library into a "-module". */ _registerCkFutures(); _registerCkArray(); _registerLBDatabase(); _registerMetaBalancer(); _registerCkCallback(); _registertempo(); _registerwaitqd(); _registerCkCheckpoint(); #if CMK_MEM_CHECKPOINT _registerCkMemCheckpoint(); #endif /* Setup Control Point Automatic Tuning Framework. By default it is enabled as a part of charm, however it won't enable its tracing module unless a +CPEnableMeasurements command line argument is specified. See trace-common.C for more info Thus there should be no noticable overhead to always having the control point framework linked in. */ #if CMK_WITH_CONTROLPOINT _registerPathHistory(); _registerControlPoints(); _registerTraceControlPoints(); #endif /** CkRegisterMainModule is generated by the (unique) "mainmodule" .ci file. It will include calls to register all the .ci files. */ CkRegisterMainModule(); /** _registerExternalModules is actually generated by charmc at link time (as "moduleinit<pid>.C"). This generated routine calls the _register functions for the .ci files of libraries linked using "-module". This funny initialization is most useful for AMPI/FEM programs, which don't have a .ci file and hence have no other way to control the _register process. */ _registerExternalModules(argv); _registerDone(); } /* The following will happen on every virtual processor in BigEmulator, not just on once per real processor */ if (CkMyRank() == 0) { CpdBreakPointInit(); } CmiNodeAllBarrier(); // Execute the initcalls registered in modules _initCallTable.enumerateInitCalls(); #if CMK_CHARMDEBUG CpdFinishInitialization(); #endif //CmiNodeAllBarrier(); CkpvAccess(_myStats) = new Stats(); CkpvAccess(_msgPool) = new MsgPool(); CmiNodeAllBarrier(); #if !(__FAULT__) CmiBarrier(); CmiBarrier(); CmiBarrier(); #endif #if CMK_SMP_TRACE_COMMTHREAD _TRACE_BEGIN_COMPUTATION(); #else if (!inCommThread) { _TRACE_BEGIN_COMPUTATION(); } #endif #ifdef ADAPT_SCHED_MEM if(CkMyRank()==0){ memCriticalEntries = new int[numMemCriticalEntries]; int memcnt=0; for(int i=0; i<_entryTable.size(); i++){ if(_entryTable[i]->isMemCritical){ memCriticalEntries[memcnt++] = i; } } } #endif #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) _messageLoggingInit(); #endif #ifndef __BIGSIM__ /* FAULT_EVAC */ CpvAccess(_validProcessors) = new char[CkNumPes()]; for(int vProc=0;vProc<CkNumPes();vProc++){ CpvAccess(_validProcessors)[vProc]=1; } _ckEvacBcastIdx = CkRegisterHandler((CmiHandler)_ckEvacBcast); _ckAckEvacIdx = CkRegisterHandler((CmiHandler)_ckAckEvac); #endif CkpvAccess(startedEvac) = 0; CpvAccess(serializer) = 0; evacuate = 0; CcdCallOnCondition(CcdSIGUSR1,(CcdVoidFn)CkDecideEvacPe,0); #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) CcdCallOnCondition(CcdSIGUSR2,(CcdVoidFn)CkMlogRestart,0); #endif if(_raiseEvac){ processRaiseEvacFile(_raiseEvacFile); /* if(CkMyPe() == 2){ // CcdCallOnConditionKeep(CcdPERIODIC_10s,(CcdVoidFn)CkDecideEvacPe,0); CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000); } if(CkMyPe() == 3){ CcdCallFnAfter((CcdVoidFn)CkDecideEvacPe, 0, 10000); }*/ } if (CkMyRank() == 0) { TopoManager_init(); } CmiNodeAllBarrier(); if (!_replaySystem) { CkFtFn faultFunc_restart = CkRestartMain; if (faultFunc == NULL || faultFunc == faultFunc_restart) { // this is not restart from memory // these two are blocking calls for non-bigsim #if ! CMK_BIGSIM_CHARM CmiInitCPUAffinity(argv); CmiInitMemAffinity(argv); #endif } CmiInitCPUTopology(argv); #if CMK_SHARED_VARS_POSIX_THREADS_SMP if (CmiCpuTopologyEnabled()) { int *pelist; int num; CmiGetPesOnPhysicalNode(0, &pelist, &num); #if !CMK_MULTICORE && !CMK_SMP_NO_COMMTHD // Count communication threads, if present // XXX: Assuming uniformity of node size here num += num/CmiMyNodeSize(); #endif if (!_Cmi_forceSpinOnIdle && num > CmiNumCores()) { if (CmiMyPe() == 0) CmiPrintf("\nCharm++> Warning: the number of SMP threads (%d) is greater than the number of physical cores (%d), so threads will sleep while idling. Use +CmiSpinOnIdle or +CmiSleepOnIdle to control this directly.\n\n", num, CmiNumCores()); CmiLock(CksvAccess(_nodeLock)); if (! _Cmi_sleepOnIdle) _Cmi_sleepOnIdle = 1; CmiUnlock(CksvAccess(_nodeLock)); } } #endif } if(CmiMyPe() == 0) { char *topoFilename; if(CmiGetArgStringDesc(argv,"+printTopo",&topoFilename,"topo file name")) { std::stringstream sstm; sstm << topoFilename << "." << CmiMyPartition(); std::string result = sstm.str(); FILE *fp; fp = fopen(result.c_str(), "w"); if (fp == NULL) { CkPrintf("Error opening %s file, writing to stdout\n", topoFilename); fp = stdout; } TopoManager_printAllocation(fp); fclose(fp); } } #if CMK_USE_PXSHM && ( CMK_CRAYXE || CMK_CRAYXC ) && CMK_SMP // for SMP on Cray XE6 (hopper) it seems pxshm has to be initialized // again after cpuaffinity is done if (CkMyRank() == 0) { CmiInitPxshm(argv); } CmiNodeAllBarrier(); #endif //CldCallback(); #if CMK_BIGSIM_CHARM && CMK_CHARMDEBUG // Register the BG handler for CCS. Notice that this is put into a variable shared by // the whole real processor. This because converse needs to find it. We check that all // virtual processors register the same index for this handler. CpdBgInit(); #endif if (faultFunc) { #if CMK_WITH_STATS if (CkMyPe()==0) _allStats = new Stats*[CkNumPes()]; #endif if (!inCommThread) { CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0); msg->argc = CmiGetArgc(argv); msg->argv = argv; faultFunc(_restartDir, msg); CkFreeMsg(msg); } }else if(CkMyPe()==0){ #if CMK_WITH_STATS _allStats = new Stats*[CkNumPes()]; #endif register size_t i, nMains=_mainTable.size(); for(i=0;i<nMains;i++) /* Create all mainchares */ { register int size = _chareTable[_mainTable[i]->chareIdx]->size; register void *obj = malloc(size); _MEMCHECK(obj); _mainTable[i]->setObj(obj); CkpvAccess(_currentChare) = obj; CkpvAccess(_currentChareType) = _mainTable[i]->chareIdx; register CkArgMsg *msg = (CkArgMsg *)CkAllocMsg(0, sizeof(CkArgMsg), 0); msg->argc = CmiGetArgc(argv); msg->argv = argv; _entryTable[_mainTable[i]->entryIdx]->call(msg, obj); #if (defined(_FAULT_MLOG_) || defined(_FAULT_CAUSAL_)) CpvAccess(_currentObj) = (Chare *)obj; #endif } _mainDone = 1; _STATS_RECORD_CREATE_CHARE_N(nMains); _STATS_RECORD_PROCESS_CHARE_N(nMains); for(i=0;i<_readonlyMsgs.size();i++) /* Send out readonly messages */ { register void *roMsg = (void *) *((char **)(_readonlyMsgs[i]->pMsg)); if(roMsg==0) continue; //Pack the message and send it to all other processors register envelope *env = UsrToEnv(roMsg); env->setSrcPe(CkMyPe()); env->setMsgtype(ROMsgMsg); env->setRoIdx(i); CmiSetHandler(env, _initHandlerIdx); CkPackMessage(&env); CmiSyncBroadcast(env->getTotalsize(), (char *)env); CpvAccess(_qd)->create(CkNumPes()-1); //For processor 0, unpack and re-set the global CkUnpackMessage(&env); _processROMsgMsg(env); _numInitMsgs++; } //Determine the size of the RODataMessage PUP::sizer ps; for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(ps); //Allocate and fill out the RODataMessage envelope *env = _allocEnv(RODataMsg, ps.size()); PUP::toMem pp((char *)EnvToUsr(env)); for(i=0;i<_readonlyTable.size();i++) _readonlyTable[i]->pupData(pp); env->setCount(++_numInitMsgs); env->setSrcPe(CkMyPe()); CmiSetHandler(env, _initHandlerIdx); DEBUGF(("[%d,%.6lf] RODataMsg being sent of size %d \n",CmiMyPe(),CmiWallTimer(),env->getTotalsize())); CmiSyncBroadcastAndFree(env->getTotalsize(), (char *)env); CpvAccess(_qd)->create(CkNumPes()-1); _initDone(); } DEBUGF(("[%d,%d%.6lf] inCommThread %d\n",CmiMyPe(),CmiMyRank(),CmiWallTimer(),inCommThread)); // when I am a communication thread, I don't participate initDone. if (inCommThread) { CkNumberHandlerEx(_bocHandlerIdx,(CmiHandlerEx)_processHandler, CkpvAccess(_coreState)); CkNumberHandlerEx(_charmHandlerIdx,(CmiHandlerEx)_processHandler , CkpvAccess(_coreState)); _processBufferedMsgs(); } #if CMK_CHARMDEBUG // Should not use CpdFreeze inside a thread (since this processor is really a user-level thread) if (CpvAccess(cpdSuspendStartup)) { //CmiPrintf("In Parallel Debugging mode .....\n"); CpdFreeze(); } #endif #if __FAULT__ if(killFlag){ readKillFile(); } #endif }
static void CmiStartThreads(char **argv) { pthread_t pid; size_t i; int ok, tocreate; pthread_attr_t attr; int start, end; MACHSTATE(4,"CmiStartThreads") CmiMemLock_lock=CmiCreateLock(); comm_mutex=CmiCreateLock(); _smp_mutex = CmiCreateLock(); #if defined(CMK_NO_ASM_AVAILABLE) && CMK_PCQUEUE_LOCK cmiMemoryLock = CmiCreateLock(); if (CmiMyNode()==0) printf("Charm++ warning> fences and atomic operations not available in native assembly\n"); #endif #if ! (CMK_HAS_TLS_VARIABLES && !CMK_NOT_USE_TLS_THREAD) pthread_key_create(&Cmi_state_key, 0); Cmi_state_vector = (CmiState)calloc(_Cmi_mynodesize+1, sizeof(struct CmiStateStruct)); for (i=0; i<_Cmi_mynodesize; i++) CmiStateInit(i+Cmi_nodestart, i, CmiGetStateN(i)); /*Create a fake state structure for the comm. thread*/ /* CmiStateInit(-1,_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); */ CmiStateInit(_Cmi_mynode+CmiNumPes(),_Cmi_mynodesize,CmiGetStateN(_Cmi_mynodesize)); #else /* for main thread */ Cmi_state_vector = (CmiState *)calloc(_Cmi_mynodesize+1, sizeof(CmiState)); #if CMK_CONVERSE_MPI /* main thread is communication thread */ if(!CharmLibInterOperate) { CmiStateInit(_Cmi_mynode+CmiNumPes(), _Cmi_mynodesize, &Cmi_mystate); Cmi_state_vector[_Cmi_mynodesize] = &Cmi_mystate; } else #endif { /* main thread is of rank 0 */ CmiStateInit(Cmi_nodestart, 0, &Cmi_mystate); Cmi_state_vector[0] = &Cmi_mystate; } #endif #if CMK_MULTICORE || CMK_SMP_NO_COMMTHD if (!Cmi_commthread) tocreate = _Cmi_mynodesize-1; else #endif tocreate = _Cmi_mynodesize; #if CMK_CONVERSE_MPI if(!CharmLibInterOperate) { start = 0; end = tocreate - 1; /* skip comm thread */ } else #endif { start = 1; end = tocreate; /* skip rank 0 main thread */ } for (i=start; i<=end; i++) { pthread_attr_init(&attr); pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); ok = pthread_create(&pid, &attr, call_startfn, (void *)i); if (ok<0) PerrorExit("pthread_create"); pthread_attr_destroy(&attr); } #if ! (CMK_HAS_TLS_VARIABLES && !CMK_NOT_USE_TLS_THREAD) #if CMK_CONVERSE_MPI if(!CharmLibInterOperate) pthread_setspecific(Cmi_state_key, Cmi_state_vector+_Cmi_mynodesize); else #endif pthread_setspecific(Cmi_state_key, Cmi_state_vector); #endif MACHSTATE(4,"CmiStartThreads done") }
void CmiInitCPUAffinity(char **argv) { static skt_ip_t myip; int ret, i, exclude; hostnameMsg *msg; char *pemap = NULL; char *commap = NULL; char *pemapfile = NULL; int show_affinity_flag; int affinity_flag = CmiGetArgFlagDesc(argv,"+setcpuaffinity", "set cpu affinity"); while (CmiGetArgIntDesc(argv,"+excludecore", &exclude, "avoid core when setting cpuaffinity")) { if (CmiMyRank() == 0) add_exclude(exclude); affinity_flag = 1; } if (CmiGetArgStringDesc(argv, "+pemapfile", &pemapfile, "define pe to core mapping file")) { FILE *fp; char buf[128]; pemap = (char*)malloc(1024); fp = fopen(pemapfile, "r"); if (fp == NULL) CmiAbort("pemapfile does not exist"); while (!feof(fp)) { if (fgets(buf, 128, fp)) { if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0; strcat(pemap, buf); } } fclose(fp); if (CmiMyPe()==0) CmiPrintf("Charm++> read from pemap file '%s': %s\n", pemapfile, pemap); } CmiGetArgStringDesc(argv, "+pemap", &pemap, "define pe to core mapping"); if (pemap!=NULL && excludecount>0) CmiAbort("Charm++> +pemap can not be used with +excludecore.\n"); CmiGetArgStringDesc(argv, "+commap", &commap, "define comm threads to core mapping"); if (pemap!=NULL || commap!=NULL) affinity_flag = 1; show_affinity_flag = CmiGetArgFlagDesc(argv,"+showcpuaffinity", "print cpu affinity"); cpuAffinityHandlerIdx = CmiRegisterHandler((CmiHandler)cpuAffinityHandler); cpuAffinityRecvHandlerIdx = CmiRegisterHandler((CmiHandler)cpuAffinityRecvHandler); if (CmiMyRank() ==0) { affLock = CmiCreateLock(); } #if CMK_BLUEGENEP || CMK_BLUEGENEQ if(affinity_flag){ affinity_flag = 0; if(CmiMyPe()==0) CmiPrintf("Charm++> cpu affinity setting is not needed on Blue Gene, thus ignored.\n"); } if(show_affinity_flag){ show_affinity_flag = 0; if(CmiMyPe()==0) CmiPrintf("Charm++> printing cpu affinity is not supported on Blue Gene.\n"); } #endif if (!affinity_flag) { if (show_affinity_flag) CmiPrintCPUAffinity(); return; } if (CmiMyPe() == 0) { CmiPrintf("Charm++> cpu affinity enabled. \n"); if (excludecount > 0) { CmiPrintf("Charm++> cpuaffinity excludes core: %d", excludecore[0]); for (i=1; i<excludecount; i++) CmiPrintf(" %d", excludecore[i]); CmiPrintf(".\n"); } if (pemap!=NULL) CmiPrintf("Charm++> cpuaffinity PE-core map : %s\n", pemap); } if (CmiMyPe() >= CmiNumPes()) { /* this is comm thread */ /* comm thread either can float around, or pin down to the last rank. however it seems to be reportedly slower if it is floating */ CmiNodeAllBarrier(); if (commap != NULL) { int mycore = search_pemap(commap, CmiMyPeGlobal()-CmiNumPesGlobal()); if(CmiMyPe()-CmiNumPes()==0) printf("Charm++> set comm %d on node %d to core #%d\n", CmiMyPe()-CmiNumPes(), CmiMyNode(), mycore); if (-1 == CmiSetCPUAffinity(mycore)) CmiAbort("set_cpu_affinity abort!"); CmiNodeAllBarrier(); if (show_affinity_flag) CmiPrintCPUAffinity(); return; /* comm thread return */ } else { /* if (CmiSetCPUAffinity(CmiNumCores()-1) == -1) CmiAbort("set_cpu_affinity abort!"); */ #if !CMK_CRAYXT && !CMK_CRAYXE && !CMK_CRAYXC && !CMK_BLUEGENEQ if (pemap == NULL) { #if CMK_MACHINE_PROGRESS_DEFINED while (affinity_doneflag < CmiMyNodeSize()) CmiNetworkProgress(); #else #if CMK_SMP #error "Machine progress call needs to be implemented for cpu affinity!" #endif #endif } #endif #if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC /* if both pemap and commmap are NULL, will compute one */ if (pemap != NULL) #endif { CmiNodeAllBarrier(); if (show_affinity_flag) CmiPrintCPUAffinity(); return; /* comm thread return */ } } } if (pemap != NULL && CmiMyPe()<CmiNumPes()) { /* work thread */ int mycore = search_pemap(pemap, CmiMyPeGlobal()); if(show_affinity_flag) CmiPrintf("Charm++> set PE %d on node %d to core #%d\n", CmiMyPe(), CmiMyNode(), mycore); if (mycore >= CmiNumCores()) { CmiPrintf("Error> Invalid core number %d, only have %d cores (0-%d) on the node. \n", mycore, CmiNumCores(), CmiNumCores()-1); CmiAbort("Invalid core number"); } if (CmiSetCPUAffinity(mycore) == -1) CmiAbort("set_cpu_affinity abort!"); CmiNodeAllBarrier(); CmiNodeAllBarrier(); /* if (show_affinity_flag) CmiPrintCPUAffinity(); */ return; } #if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC { int numCores = CmiNumCores(); int myid = getXTNodeID(CmiMyNodeGlobal(), CmiNumNodesGlobal()); int myrank; int pe, mype = CmiMyPeGlobal(); int node = CmiMyNodeGlobal(); int nnodes = 0; #if CMK_SMP if (CmiMyPe() >= CmiNumPes()) { /* this is comm thread */ int node = CmiMyPe() - CmiNumPes(); mype = CmiGetPeGlobal(CmiNodeFirst(node) + CmiMyNodeSize() - 1, CmiMyPartition()); /* last pe on SMP node */ node = CmiGetNodeGlobal(node, CmiMyPartition()); } #endif pe = mype - 1; while (pe >= 0) { int n = CmiNodeOf(pe); if (n != node) { nnodes++; node = n; } if (getXTNodeID(n, CmiNumNodesGlobal()) != myid) break; pe --; } CmiAssert(numCores > 0); myrank = (mype - pe - 1 + nnodes)%numCores; #if CMK_SMP if (CmiMyPe() >= CmiNumPes()) myrank = (myrank + 1)%numCores; #endif if (-1 != CmiSetCPUAffinity(myrank)) { DEBUGP(("Processor %d is bound to core #%d on node #%d\n", CmiMyPe(), myrank, mynode)); } else{ CmiPrintf("Processor %d set affinity failed!\n", CmiMyPe()); CmiAbort("set cpu affinity abort!\n"); } } if (CmiMyPe() < CmiNumPes()) CmiNodeAllBarrier(); CmiNodeAllBarrier(); #else /* get my ip address */ if (CmiMyRank() == 0) { #if CMK_HAS_GETHOSTNAME myip = skt_my_ip(); /* not thread safe, so only calls on rank 0 */ #else CmiAbort("Can not get unique name for the compute nodes. \n"); #endif } CmiNodeAllBarrier(); /* prepare a msg to send */ msg = (hostnameMsg *)CmiAlloc(sizeof(hostnameMsg)); CmiSetHandler((char *)msg, cpuAffinityHandlerIdx); msg->pe = CmiMyPe(); msg->ip = myip; msg->ncores = CmiNumCores(); DEBUGP(("PE %d's node has %d number of cores. \n", CmiMyPe(), msg->ncores)); msg->rank = 0; CmiSyncSendAndFree(0, sizeof(hostnameMsg), (void *)msg); if (CmiMyPe() == 0) { int i; hostTable = CmmNew(); rankmsg = (rankMsg *)CmiAlloc(sizeof(rankMsg)+CmiNumPes()*sizeof(int)*2); CmiSetHandler((char *)rankmsg, cpuAffinityRecvHandlerIdx); rankmsg->ranks = (int *)((char*)rankmsg + sizeof(rankMsg)); rankmsg->nodes = (int *)((char*)rankmsg + sizeof(rankMsg) + CmiNumPes()*sizeof(int)); for (i=0; i<CmiNumPes(); i++) { rankmsg->ranks[i] = 0; rankmsg->nodes[i] = -1; } for (i=0; i<CmiNumPes(); i++) CmiDeliverSpecificMsg(cpuAffinityHandlerIdx); } /* receive broadcast from PE 0 */ CmiDeliverSpecificMsg(cpuAffinityRecvHandlerIdx); CmiLock(affLock); affinity_doneflag++; CmiUnlock(affLock); CmiNodeAllBarrier(); #endif if (show_affinity_flag) CmiPrintCPUAffinity(); }