void vars_check_ctv_privacy(vars_chare c)
{
int me = (size_t)CthSelf();
CtvAccess(ctv1) = me;
vars_ack(c);
CthSuspend();
if (CtvAccess(ctv1) != me) {
CmiPrintf("ctv privacy test failed.\n");
exit(1);
}
vars_ack(c);
CthFree(CthSelf());
CthSuspend();
}
/// If we're waiting for any pages, suspend our thread.
void MSA_Thread_Listener::suspend(void) {
bool verbose=false;
if (count>0) {
thread=CthSelf();
if (verbose) CkPrintf("Thread %p suspending for %d signals\n",
CthSelf(),count);
CthSuspend();
if (verbose) CkPrintf("Thread %p resumed\n",CthSelf());
}
}
void waitqd_QDChare::waitQD(void) {
if (waitStarted == 1) {
CdsFifo_Enqueue((CdsFifo)threadList, (void *)CthSelf());
} else {
waitStarted = 1;
threadList = (void*) CdsFifo_Create();
CdsFifo_Enqueue((CdsFifo) threadList, (void *)CthSelf());
CkStartQD(CkIndex_waitqd_QDChare::onQD((CkQdMsg*)0), &thishandle);
}
CthSuspend();
}
CpmInvokable vars_control()
{
struct vars_chare c; CthThread t1,t2;
t1 = CthCreate(vars_check_ctv_privacy, (void *)&c, 0);
t2 = CthCreate(vars_check_ctv_privacy, (void *)&c, 0);
CthSetStrategyDefault(t1);
CthSetStrategyDefault(t2);
CthAwaken(t1); CthAwaken(t2);
c.countdown = 2; c.pending = CthSelf(); CthSuspend();
CthAwaken(t1); CthAwaken(t2);
c.countdown = 2; c.pending = CthSelf(); CthSuspend();
Cpm_vars_set_cpv_and_csv(CpmSend(CpmALL), &c);
c.countdown = CmiNumPes(); c.pending = CthSelf(); CthSuspend();
Cpm_vars_check_cpv_and_csv(CpmSend(CpmALL), &c);
c.countdown = CmiNumPes(); c.pending = CthSelf(); CthSuspend();
Cpm_megacon_ack(CpmSend(0));
}
void LdbCoordinator::rebalance(Controller *c)
{
if (Node::Object()->simParameters->ldBalancer == LDBAL_NONE)
return;
iout << "LDB: ============= START OF LOAD BALANCING ============== " << CmiWallTimer() << "\n" << endi;
DebugM(3, "Controller reached load balance barrier.\n");
controllerReported = 1;
controllerThread = c;
CProxy_LdbCoordinator(thisgroup).barrier();
CthSuspend();
}
// require the data from a thread
void ReductionMgr::require(RequireReduction* handle) {
int setID = handle->reductionSetID;
ReductionSet *set = reductionSets[setID];
int seqNum = handle->sequenceNumber;
ReductionSetData *data = set->getData(seqNum);
if ( data->submitsRecorded < set->submitsRegistered ) {
set->threadIsWaiting = 1;
set->waitingForSequenceNumber = seqNum;
set->waitingThread = CthSelf();
//iout << "seq " << seqNum << " waiting\n" << endi;
CthSuspend();
}
set->threadIsWaiting = 0;
//iout << "seq " << seqNum << " consumed\n" << endi;
delete handle->currentData;
handle->currentData = set->removeData(seqNum);
handle->data = handle->currentData->data;
handle->sequenceNumber = ++seqNum;
}
/**
threadInfo methods
*/
void commThreadInfo::run()
{
CpvAccess(CthResumeBigSimThreadIdx) = BgRegisterHandler((BgHandler)CthResumeNormalThread);
tSTARTTIME = CmiWallTimer();
if (!tSTARTED) {
tSTARTED = 1;
// InitHandlerTable();
BgNodeStart(BgGetArgc(), BgGetArgv());
/* bnv should be initialized */
}
threadQueue *commQ = myNode->commThQ;
//int recvd=0; //for debugging only
for (;;) {
char *msg = getFullBuffer();
if (!msg) {
// tCURRTIME += (CmiWallTimer()-tSTARTTIME);
commQ->enq(CthSelf());
DEBUGF(("[%d] comm thread suspend.\n", BgMyNode()));
CthSuspend();
DEBUGF(("[%d] comm thread assume.\n", BgMyNode()));
// tSTARTTIME = CmiWallTimer();
continue;
}
DEBUGF(("[%d] comm thread has a msg.\n", BgMyNode()));
//printf("on node %d, comm thread process a msg %p with type %d\n", BgMyNode(), msg, CmiBgMsgType(msg));
/* schedule a worker thread, if small work do it itself */
if (CmiBgMsgType(msg) == SMALL_WORK) {
if (CmiBgMsgRecvTime(msg) > tCURRTIME) tCURRTIME = CmiBgMsgRecvTime(msg);
// tSTARTTIME = CmiWallTimer();
/* call user registered handler function */
BgProcessMessage(this, msg);
}
else {
#if BIGSIM_TIMING
correctMsgTime(msg);
#endif
//recvd++;
//DEBUGM(4, ("[N%d] C[%d] will add a msg (handler=%d | cnt=%d", BgMyNode(), id, CmiBgMsgHandle(msg), recvd));
int msgLen = CmiBgMsgLength(msg);
DEBUGM(4, (" | len: %d | type: %d | node id: %d | src pe: %d\n" , msgLen, CmiBgMsgType(msg), CmiBgMsgNodeID(msg), CmiBgMsgSrcPe(msg)));
if (CmiBgMsgThreadID(msg) == ANYTHREAD) {
DEBUGF(("anythread, call addBgNodeMessage\n"));
addBgNodeMessage(msg); /* non-affinity message */
DEBUGM(4, ("The message is added to node\n\n"));
}
else {
DEBUGF(("[N%d] affinity msg, call addBgThreadMessage to tID:%d\n",
BgMyNode(), CmiBgMsgThreadID(msg)));
addBgThreadMessage(msg, CmiBgMsgThreadID(msg));
DEBUGM(4, ("The message is added to thread(%d)\n\n", CmiBgMsgThreadID(msg)));
}
}
/* let other communication thread do their jobs */
// tCURRTIME += (CmiWallTimer()-tSTARTTIME);
if (!schedule_flag) CthYield();
tSTARTTIME = CmiWallTimer();
}
}
//The original version of scheduler
void workThreadInfo::scheduler(int count)
{
ckMsgQueue &q1 = myNode->nodeQ;
ckMsgQueue &q2 = myNode->affinityQ[id];
int cycle = CsdStopFlag;
int recvd = 0;
for (;;) {
char *msg=NULL;
int e1 = q1.isEmpty();
int e2 = q2.isEmpty();
int fromQ2 = 0; // delay the deq of msg from affinity queue
// not deq from nodeQ assuming no interrupt in the handler
if (e1 && !e2) { msg = q2[0]; fromQ2 = 1;}
// else if (e2 && !e1) { msg = q1.deq(); }
else if (e2 && !e1) { msg = q1[0]; }
else if (!e1 && !e2) {
if (CmiBgMsgRecvTime(q1[0]) < CmiBgMsgRecvTime(q2[0])) {
// msg = q1.deq();
msg = q1[0];
}
else {
msg = q2[0];
fromQ2 = 1;
}
}
/* if no msg is ready, go back to sleep */
if ( msg == NULL ) {
// tCURRTIME += (CmiWallTimer()-tSTARTTIME);
DEBUGM(4,("N[%d] work thread %d has no msg and go to sleep!\n", BgMyNode(), id));
if (watcher) watcher->replay();
#if BIGSIM_OUT_OF_CORE && BIGSIM_OOC_PREFETCH
if(bgUseOutOfCore){
//thread scheduling point!!
workThreadInfo *thisThd = schedWorkThds->pop();
//CmiPrintf("thisThd=%p, actualThd=%p, equal=%d qsize=%d\n", thisThd, this, thisThd==this, schedWorkThds->size());
assert(thisThd==this);
}
#endif
CthSuspend();
DEBUGM(4, ("N[%d] work thread %d awakened!\n", BgMyNode(), id));
continue;
}
#if BIGSIM_TIMING
correctMsgTime(msg);
#if THROTTLE_WORK
if (correctTimeLog) {
if (CmiBgMsgRecvTime(msg) > gvt+ BG_LEASH) {
double nextT = CmiBgMsgRecvTime(msg);
int prio = (int)(nextT*PRIO_FACTOR)+1;
if (prio < 0) {
CmiPrintf("PRIO_FACTOR %e is too small. \n", PRIO_FACTOR);
CmiAbort("BigSim time correction abort!\n");
}
//CmiPrintf("Thread %d YieldPrio: %g gvt: %g leash: %g\n", id, nextT, gvt, BG_LEASH);
CthYieldPrio(CQS_QUEUEING_IFIFO, sizeof(int), (unsigned int*)&prio);
continue;
}
}
#endif
#endif /* TIMING */
DEBUGM(2, ("[N%d] work thread T%d has a msg with recvT:%e msgId:%d.\n", BgMyNode(), id, CmiBgMsgRecvTime(msg), CmiBgMsgID(msg)));
//if (tMYNODEID==0)
//CmiPrintf("[%d] recvT: %e\n", tMYNODEID, CmiBgMsgRecvTime(msg));
if (CmiBgMsgRecvTime(msg) > currTime) {
tCURRTIME = CmiBgMsgRecvTime(msg);
}
#if 1
if (fromQ2 == 1) q2.deq();
else q1.deq();
#endif
recvd ++;
DEBUGM(4, ("[N%d] W[%d] will process a msg (handler=%d | cnt=%d", BgMyNode(), id, CmiBgMsgHandle(msg), recvd));
int msgLen = CmiBgMsgLength(msg);
DEBUGM(4, (" | len: %d | type: %d | node id: %d | src pe: %d\n" , msgLen, CmiBgMsgType(msg), CmiBgMsgNodeID(msg), CmiBgMsgSrcPe(msg)));
for(int msgIndex=CmiBlueGeneMsgHeaderSizeBytes-1; msgIndex<msgLen; msgIndex++)
DEBUGM(2, ("%d,", msg[msgIndex]));
DEBUGM(2,("\n"));
DEBUGM(4, ("[N%d] W[%d] now has %d msgs from own queue and %d from affinity before processing msg\n", BgMyNode(), id, q1.length(), q2.length()));
//CmiMemoryCheck();
// BgProcessMessage may trap into scheduler
if(bgUseOutOfCore){
#if 0
if(startOutOfCore){
DEBUGM(4, ("to execute in ooc mode\n"));
if(isCoreOnDisk) this->broughtIntoMem();
BgProcessMessage(this, msg); //startOutOfCore may be changed in processing this msg (AMPI_Init)
if(startOOCChanged){
//indicate AMPI_Init is called and before it is finished, out-of-core is not executed
//just to track the 0->1 change phase (which means MPI_Init is finished)
//the 1->0 phase is not tracked because "startOutOfCore" is unset so that
//the next processing of a msg will not go into this part of code
startOOCChanged=0;
}else{
//if(!isCoreOnDisk) { //the condition is added for virtual process
this->takenOutofMem();
//}
}
}else{
DEBUGM(4, ("to execute not in ooc mode\n"));
if(isCoreOnDisk) {
CmiAbort("This should never be entered!\n");
this->broughtIntoMem();
}
//put before processing msg since thread may be scheduled during processing the msg
BgProcessMessage(this, msg);
}
#else
//schedWorkThds->print();
bgOutOfCoreSchedule(this);
BG_ENTRYSTART(msg);
#if BIGSIM_OOC_PREFETCH
#if !BIGSIM_OOC_NOPREFETCH_AIO
//do prefetch here for the next different thread in queue (schedWorkThds)
assert(schedWorkThds->peek(0)==this);
for(int offset=1; offset<schedWorkThds->size(); offset++) {
workThreadInfo *nThd = schedWorkThds->peek(offset);
//if nThd's core has been dumped to disk, then we could prefetch its core.
//otherwise, it is the first time for the thread to process a message, thus
//no need to resort to disk to find its core
if(nThd!=this && !checkThreadInCore(nThd)
&& nThd->isCoreOnDisk && oocPrefetchSpace->occupiedThd==NULL) {
oocPrefetchSpace->newPrefetch(nThd);
}
}
#endif
#endif
BgProcessMessage(this, msg);
#endif
}else{
DEBUGM(4, ("to execute not in ooc mode\n"));
BG_ENTRYSTART(msg);
BgProcessMessage(this, msg);
}
DEBUGM(4, ("[N%d] W[%d] now has %d msgs from own queue and %d from affinity after processing msg\n\n", BgMyNode(), id, q1.length(), q2.length()));
BG_ENTRYEND();
// counter of processed real mesgs
stateCounters.realMsgProcCnt++;
// NOTE: I forgot why I delayed the dequeue after processing it
#if 0
if (fromQ2 == 1) q2.deq();
else q1.deq();
#endif
//recvd ++;
//DEBUGF(("[N%d] work thread T%d finish a msg.\n", BgMyNode(), id));
//CmiPrintf("[N%d] work thread T%d finish a msg (msg=%s, cnt=%d).\n", BgMyNode(), id, msg, recvd);
if ( recvd == count) return;
if (cycle != CsdStopFlag) break;
/* let other work thread do their jobs */
if (schedule_flag) {
DEBUGF(("[N%d] work thread T%d suspend when done - %d to go.\n", BgMyNode(), tMYID, q2.length()));
CthSuspend();
DEBUGF(("[N%d] work thread T%d awakened here.\n", BgMyNode(), id));
}
else {
#if BIGSIM_OUT_OF_CORE && BIGSIM_OOC_PREFETCH
//thread scheduling point!!
//Suspend and put itself back to the end of the queue
if(bgUseOutOfCore){
workThreadInfo *thisThd = schedWorkThds->pop();
//CmiPrintf("thisThd=%p, actualThd=%p, equal=%d qsize=%d\n", thisThd, this, thisThd==this, schedWorkThds->size());
assert(thisThd==this);
schedWorkThds->push(this);
}
#endif
CthYield();
}
}
CsdStopFlag --;
}
void barrier() {
contribute (CkCallback(CkReductionTarget(Worker, barrierH), workerarray));
t = CthSelf();
CthSuspend();
}
