void threadInfo::takenOutofMem(){ DEBUGM(5, ("=====[N%d] work thread T[%d] outof mem=====.\n", BgMyNode(), id)); //CmiPrintStackTrace(0); assert(isCoreOnDisk==0); _BgOutOfCoreFlag=1; const char *dirname = "/tmp/CORE"; //every body make dir in case it is local directory CmiMkdir(dirname); char filename[128]; sprintf(filename, "%s/%d.dat", dirname, globalId); FILE* fp = fopen(filename, "wb"); if(fp==NULL){ printf("Error: %s cannot be opened when bringing thread %d to core\n", filename, globalId); return; } //_BgOutOfCoreFlag=1; PUP::toDisk p(fp); //out-of-core is not a real migration, so turn off the notifyListener option p.becomeDeleting(); #if BIGSIM_OUT_OF_CORE CkPupArrayElementsData(p, 0); #endif CmiUInt8 fsize = 0; fseek(fp, 0, SEEK_END); fsize = ftell(fp); //set this thread's memory usage memUsed = fsize/1024.0/1024.0; fflush(fp); fclose(fp); DEBUGM(6,("Before removing array elements on proc[%d]\n", globalId)); #if BIGSIM_OUT_OF_CORE CkRemoveArrayElements(); #endif _BgOutOfCoreFlag=0; isCoreOnDisk=1; //printf("mem usage after thread %d out: %fMB\n", globalId, CmiMemoryUsage()/1024.0/1024.0); #if BIGSIM_OOC_PREFETCH thdsOOCPreStatus[preStsIdx].bufsize= fsize; #endif }
int addEventPv( acqEvent_t *ev, char *name, int norecord, char *fmt, acqPvReady_t ready, int isSpectrum) { int idx; int cnt; int nbytes; if( name == NULL || ! *name ) /* don't add null PV's */ return -1; idx = ev->numPvList; nbytes = (1+ev->numPvList)* sizeof (*ev->pvList); // Added %ld to address 64 bit issue (David Chevrier, Oct 27 2011) DEBUGM(ev->master,1) printf("adding event %d total bytes %d (%d*%ld)\n", idx, nbytes, ev->numPvList+1, sizeof(*ev->pvList)); ev->pvList = realloc(ev->pvList, nbytes); ev->numPvList++; ev->pvList[idx].pvName = strdup( name); ev->pvList[idx].noRecord = norecord; ev->pvList[idx].ready = ready; ev->pvList[idx].isSpectrum = isSpectrum; ev->pvList[idx].displayRow = -1; ev->pvList[idx].noConnect = NCR_IGNORE; if( fmt && *fmt) ev->pvList[idx].outputFormat = strdup( fmt); else ev->pvList[idx].outputFormat = NULL; /* RU! I'm not sure you should have added this. Consider taking it back out.*/ /* Glen: this allows passing the pvList directly as a callback, and then allows referencing * the parent event structure of the PV */ for( cnt=0; cnt < ev->numPvList; cnt++) ev->pvList[cnt].event = ev; return idx; }
virtual void evaluate( const FunctionNode& function ) { return; return; switch (function.getType()) { case FunctionNode::FT_MOVE : { //suma // int result = 0; // for (int i = 0; i < arity - 1; ++i) { // result += arguments[i]->getValue(); // } // arguments[arity - 1]->setValue(result); // std::cout << function.toString() << std::endl; break; } case FunctionNode::FT_SWAP : { //resta // int result = 0; // for (int i = 0; i < arity - 1; ++i) { // result -= arguments[i]->getValue(); // } // arguments[arity - 1]->setValue(result); break; } default: DEBUGM(TO_STRING(function.getType())); assert(0); } }
static int build_scanRecord_links(acqScan_t *sc) { int i; char *controlPV = NULL; DEBUGM(sc->master,1) printf("build_scanRecord_links\n"); if( sc->acqControlList) { for(i=0; i < sc->numControlPV; i++) { if( sc->acqControlList[i].controlPV) controlPV = sc->acqControlList[i].controlPV; if( !controlPV) /* should not happen: 'valid settings' checks this */ return 0; sc->acqControlList[i].controlChan = build_connector( NULL, controlPV); sc->acqControlList[i].controlChan->user_data = (void *) sc; sc->acqControlList[i].controlChan->update = NULL; /* set to callbacks for updates */ } } for( i=0; i < NUM_SCAN_TRIGGERS; i++) { acqAction_t *ap; for( ap=sc->actions[i]; ap ; ap=(ap->next==sc->actions[i]?NULL:ap->next) ) { if( buildAction( ap) == 0) { sc->master->messageAdd(sc->master, "can't link action %s type %d", tgTypeName(i), ap->type); return 0; } } } sc->startEvent = epicsEventCreate(0); sc->endEvent = epicsEventCreate(0); sc->putNotify = epicsEventCreate(0); if( sc->startEvent == 0 || sc->endEvent == 0) return 0; return 1; }
/* * move from 'stop' to 'standby'. * return 1 on success, 0 if unable to set up. */ int Standby_mode( acqMaster_t *master) { acqScan_t *sc; acqEvent_t *ev; char *errMsg; int valid = 1; int nManualTrigger = 0; if( master->GlobalContext == NULL) { init_CA(); master->GlobalContext = ca_current_context(); } master->messageAdd(master, "Standby Mode"); master->putMode( master, AS_STARTUP); /* * verify content for all records */ for( sc = first_acqScan(master); sc ; sc = next_acqScan( sc) ) { if ( ( errMsg = valid_scanRecord_settings(sc) ) != NULL ) { master->messageAdd(master, "Scan %s not valid: %s", sc->scanName, errMsg); valid = 0; } if( sc->useStart) nManualTrigger++; } if( !valid) { master->putMode( master, AS_OFF); return 0; } if( nManualTrigger == 0) { master->messageAdd(master, "No Scans set for 'Trigger on Start'"); return 0; } for( ev= first_acqEvent(master); ev ; ev = next_acqEvent( ev) ) { if( ( errMsg = valid_eventRecord_settings(ev) ) != NULL ) { master->messageAdd(master, "Event %s not valid: %s", ev->eventName, errMsg); valid = 0; } } if( !valid) { master->putMode(master, AS_OFF); return 0; } DEBUGM(master,1) printf("Standby mode: building links\n"); /* * establish all links. on failure, clean up and return */ for( sc = first_acqScan(master); sc ; sc = next_acqScan(sc) ) { if( build_scanRecord_links(sc) != 1) { master->messageAdd(master, "Linking for scan '%s' failed", sc->scanName); valid = 0; break; } } for( ev = first_acqEvent(master); ev ; ev = next_acqEvent(ev) ) { if( build_eventRecord_links( ev) != 1) { master->messageAdd(master, "Linking for event '%s' failed", ev->eventName); valid = 0; break; } } if( !valid) { /* remove the links just created */ for( sc = first_acqScan(master) ; sc ; sc = next_acqScan(sc) ) erase_scanRecord_links(sc); for( ev = first_acqEvent(master); ev ; ev = next_acqEvent(ev) ) erase_eventRecord_links(ev); master->putMode(master, AS_OFF); return 0; } master->putMode(master, AS_STANDBY); return 1; }
void threadInfo::broughtIntoMem(){ DEBUGM(5, ("=====[N%d] work thread T[%d] into mem=====.\n", BgMyNode(), id)); //CmiPrintStackTrace(0); //int idx =( (workThreadInfo *)this)->preStsIdx; //CmiPrintf("pe[%d]: on node[%d] thread[%d] has bufsize %ld\n", CkMyPe(), BgMyNode(), id, thdsOOCPreStatus[idx].bufsize); assert(isCoreOnDisk==1); #if BIGSIM_OOC_PREFETCH #if !BIGSIM_OOC_NOPREFETCH_AIO if(oocPrefetchSpace->occupiedThd != this){ //in this case, the prefetch is not for this workthread _BgOutOfCoreFlag=2; char *dirname = "/tmp/CORE"; //every body make dir in case it is local directory //CmiMkdir(dirname); char filename[128]; sprintf(filename, "%s/%d.dat", dirname, globalId); FILE* fp = fopen(filename, "r"); if(fp==NULL){ printf("Error: %s cannot be opened when bringing thread %d to core\n", filename, globalId); return; } //_BgOutOfCoreFlag=2; PUP::fromDisk p(fp); //out-of-core is not a real migration, so turn off the notifyListener option #if BIGSIM_OUT_OF_CORE CkPupArrayElementsData(p, 0); #endif fclose(fp); }else #endif { _BgOutOfCoreFlag=2; workThreadInfo *wthd = (workThreadInfo *)this; #if BIGSIM_OOC_NOPREFETCH_AIO oocPrefetchSpace->newPrefetch(wthd); #endif while(thdsOOCPreStatus[wthd->preStsIdx].isPrefetchFinished==0); PUP::fromMem p(oocPrefetchSpace->bufspace); #if BIGSIM_OUT_OF_CORE CkPupArrayElementsData(p, 0); #endif oocPrefetchSpace->resetPrefetch(); } #else //not doing prefetch optimization for ooc emulation _BgOutOfCoreFlag=2; const char *dirname = "/tmp/CORE"; //every body make dir in case it is local directory //CmiMkdir(dirname); char filename[128]; sprintf(filename, "%s/%d.dat", dirname, globalId); FILE* fp = fopen(filename, "r"); if(fp==NULL){ printf("Error: %s cannot be opened when bringing thread %d to core\n", filename, globalId); return; } //_BgOutOfCoreFlag=2; PUP::fromDisk p(fp); //out-of-core is not a real migration, so turn off the notifyListener option #if BIGSIM_OUT_OF_CORE CkPupArrayElementsData(p, 0); #endif fclose(fp); #endif _BgOutOfCoreFlag=0; //printf("mem usage after thread %d in: %fMB\n",globalId, CmiMemoryUsage()/1024.0/1024.0); isCoreOnDisk = 0; }
/** threadInfo methods */ void commThreadInfo::run() { CpvAccess(CthResumeBigSimThreadIdx) = BgRegisterHandler((BgHandler)CthResumeNormalThread); tSTARTTIME = CmiWallTimer(); if (!tSTARTED) { tSTARTED = 1; // InitHandlerTable(); BgNodeStart(BgGetArgc(), BgGetArgv()); /* bnv should be initialized */ } threadQueue *commQ = myNode->commThQ; //int recvd=0; //for debugging only for (;;) { char *msg = getFullBuffer(); if (!msg) { // tCURRTIME += (CmiWallTimer()-tSTARTTIME); commQ->enq(CthSelf()); DEBUGF(("[%d] comm thread suspend.\n", BgMyNode())); CthSuspend(); DEBUGF(("[%d] comm thread assume.\n", BgMyNode())); // tSTARTTIME = CmiWallTimer(); continue; } DEBUGF(("[%d] comm thread has a msg.\n", BgMyNode())); //printf("on node %d, comm thread process a msg %p with type %d\n", BgMyNode(), msg, CmiBgMsgType(msg)); /* schedule a worker thread, if small work do it itself */ if (CmiBgMsgType(msg) == SMALL_WORK) { if (CmiBgMsgRecvTime(msg) > tCURRTIME) tCURRTIME = CmiBgMsgRecvTime(msg); // tSTARTTIME = CmiWallTimer(); /* call user registered handler function */ BgProcessMessage(this, msg); } else { #if BIGSIM_TIMING correctMsgTime(msg); #endif //recvd++; //DEBUGM(4, ("[N%d] C[%d] will add a msg (handler=%d | cnt=%d", BgMyNode(), id, CmiBgMsgHandle(msg), recvd)); int msgLen = CmiBgMsgLength(msg); DEBUGM(4, (" | len: %d | type: %d | node id: %d | src pe: %d\n" , msgLen, CmiBgMsgType(msg), CmiBgMsgNodeID(msg), CmiBgMsgSrcPe(msg))); if (CmiBgMsgThreadID(msg) == ANYTHREAD) { DEBUGF(("anythread, call addBgNodeMessage\n")); addBgNodeMessage(msg); /* non-affinity message */ DEBUGM(4, ("The message is added to node\n\n")); } else { DEBUGF(("[N%d] affinity msg, call addBgThreadMessage to tID:%d\n", BgMyNode(), CmiBgMsgThreadID(msg))); addBgThreadMessage(msg, CmiBgMsgThreadID(msg)); DEBUGM(4, ("The message is added to thread(%d)\n\n", CmiBgMsgThreadID(msg))); } } /* let other communication thread do their jobs */ // tCURRTIME += (CmiWallTimer()-tSTARTTIME); if (!schedule_flag) CthYield(); tSTARTTIME = CmiWallTimer(); } }
//The original version of scheduler void workThreadInfo::scheduler(int count) { ckMsgQueue &q1 = myNode->nodeQ; ckMsgQueue &q2 = myNode->affinityQ[id]; int cycle = CsdStopFlag; int recvd = 0; for (;;) { char *msg=NULL; int e1 = q1.isEmpty(); int e2 = q2.isEmpty(); int fromQ2 = 0; // delay the deq of msg from affinity queue // not deq from nodeQ assuming no interrupt in the handler if (e1 && !e2) { msg = q2[0]; fromQ2 = 1;} // else if (e2 && !e1) { msg = q1.deq(); } else if (e2 && !e1) { msg = q1[0]; } else if (!e1 && !e2) { if (CmiBgMsgRecvTime(q1[0]) < CmiBgMsgRecvTime(q2[0])) { // msg = q1.deq(); msg = q1[0]; } else { msg = q2[0]; fromQ2 = 1; } } /* if no msg is ready, go back to sleep */ if ( msg == NULL ) { // tCURRTIME += (CmiWallTimer()-tSTARTTIME); DEBUGM(4,("N[%d] work thread %d has no msg and go to sleep!\n", BgMyNode(), id)); if (watcher) watcher->replay(); #if BIGSIM_OUT_OF_CORE && BIGSIM_OOC_PREFETCH if(bgUseOutOfCore){ //thread scheduling point!! workThreadInfo *thisThd = schedWorkThds->pop(); //CmiPrintf("thisThd=%p, actualThd=%p, equal=%d qsize=%d\n", thisThd, this, thisThd==this, schedWorkThds->size()); assert(thisThd==this); } #endif CthSuspend(); DEBUGM(4, ("N[%d] work thread %d awakened!\n", BgMyNode(), id)); continue; } #if BIGSIM_TIMING correctMsgTime(msg); #if THROTTLE_WORK if (correctTimeLog) { if (CmiBgMsgRecvTime(msg) > gvt+ BG_LEASH) { double nextT = CmiBgMsgRecvTime(msg); int prio = (int)(nextT*PRIO_FACTOR)+1; if (prio < 0) { CmiPrintf("PRIO_FACTOR %e is too small. \n", PRIO_FACTOR); CmiAbort("BigSim time correction abort!\n"); } //CmiPrintf("Thread %d YieldPrio: %g gvt: %g leash: %g\n", id, nextT, gvt, BG_LEASH); CthYieldPrio(CQS_QUEUEING_IFIFO, sizeof(int), (unsigned int*)&prio); continue; } } #endif #endif /* TIMING */ DEBUGM(2, ("[N%d] work thread T%d has a msg with recvT:%e msgId:%d.\n", BgMyNode(), id, CmiBgMsgRecvTime(msg), CmiBgMsgID(msg))); //if (tMYNODEID==0) //CmiPrintf("[%d] recvT: %e\n", tMYNODEID, CmiBgMsgRecvTime(msg)); if (CmiBgMsgRecvTime(msg) > currTime) { tCURRTIME = CmiBgMsgRecvTime(msg); } #if 1 if (fromQ2 == 1) q2.deq(); else q1.deq(); #endif recvd ++; DEBUGM(4, ("[N%d] W[%d] will process a msg (handler=%d | cnt=%d", BgMyNode(), id, CmiBgMsgHandle(msg), recvd)); int msgLen = CmiBgMsgLength(msg); DEBUGM(4, (" | len: %d | type: %d | node id: %d | src pe: %d\n" , msgLen, CmiBgMsgType(msg), CmiBgMsgNodeID(msg), CmiBgMsgSrcPe(msg))); for(int msgIndex=CmiBlueGeneMsgHeaderSizeBytes-1; msgIndex<msgLen; msgIndex++) DEBUGM(2, ("%d,", msg[msgIndex])); DEBUGM(2,("\n")); DEBUGM(4, ("[N%d] W[%d] now has %d msgs from own queue and %d from affinity before processing msg\n", BgMyNode(), id, q1.length(), q2.length())); //CmiMemoryCheck(); // BgProcessMessage may trap into scheduler if(bgUseOutOfCore){ #if 0 if(startOutOfCore){ DEBUGM(4, ("to execute in ooc mode\n")); if(isCoreOnDisk) this->broughtIntoMem(); BgProcessMessage(this, msg); //startOutOfCore may be changed in processing this msg (AMPI_Init) if(startOOCChanged){ //indicate AMPI_Init is called and before it is finished, out-of-core is not executed //just to track the 0->1 change phase (which means MPI_Init is finished) //the 1->0 phase is not tracked because "startOutOfCore" is unset so that //the next processing of a msg will not go into this part of code startOOCChanged=0; }else{ //if(!isCoreOnDisk) { //the condition is added for virtual process this->takenOutofMem(); //} } }else{ DEBUGM(4, ("to execute not in ooc mode\n")); if(isCoreOnDisk) { CmiAbort("This should never be entered!\n"); this->broughtIntoMem(); } //put before processing msg since thread may be scheduled during processing the msg BgProcessMessage(this, msg); } #else //schedWorkThds->print(); bgOutOfCoreSchedule(this); BG_ENTRYSTART(msg); #if BIGSIM_OOC_PREFETCH #if !BIGSIM_OOC_NOPREFETCH_AIO //do prefetch here for the next different thread in queue (schedWorkThds) assert(schedWorkThds->peek(0)==this); for(int offset=1; offset<schedWorkThds->size(); offset++) { workThreadInfo *nThd = schedWorkThds->peek(offset); //if nThd's core has been dumped to disk, then we could prefetch its core. //otherwise, it is the first time for the thread to process a message, thus //no need to resort to disk to find its core if(nThd!=this && !checkThreadInCore(nThd) && nThd->isCoreOnDisk && oocPrefetchSpace->occupiedThd==NULL) { oocPrefetchSpace->newPrefetch(nThd); } } #endif #endif BgProcessMessage(this, msg); #endif }else{ DEBUGM(4, ("to execute not in ooc mode\n")); BG_ENTRYSTART(msg); BgProcessMessage(this, msg); } DEBUGM(4, ("[N%d] W[%d] now has %d msgs from own queue and %d from affinity after processing msg\n\n", BgMyNode(), id, q1.length(), q2.length())); BG_ENTRYEND(); // counter of processed real mesgs stateCounters.realMsgProcCnt++; // NOTE: I forgot why I delayed the dequeue after processing it #if 0 if (fromQ2 == 1) q2.deq(); else q1.deq(); #endif //recvd ++; //DEBUGF(("[N%d] work thread T%d finish a msg.\n", BgMyNode(), id)); //CmiPrintf("[N%d] work thread T%d finish a msg (msg=%s, cnt=%d).\n", BgMyNode(), id, msg, recvd); if ( recvd == count) return; if (cycle != CsdStopFlag) break; /* let other work thread do their jobs */ if (schedule_flag) { DEBUGF(("[N%d] work thread T%d suspend when done - %d to go.\n", BgMyNode(), tMYID, q2.length())); CthSuspend(); DEBUGF(("[N%d] work thread T%d awakened here.\n", BgMyNode(), id)); } else { #if BIGSIM_OUT_OF_CORE && BIGSIM_OOC_PREFETCH //thread scheduling point!! //Suspend and put itself back to the end of the queue if(bgUseOutOfCore){ workThreadInfo *thisThd = schedWorkThds->pop(); //CmiPrintf("thisThd=%p, actualThd=%p, equal=%d qsize=%d\n", thisThd, this, thisThd==this, schedWorkThds->size()); assert(thisThd==this); schedWorkThds->push(this); } #endif CthYield(); } } CsdStopFlag --; }