/**
* lapi completion handler on the recv side. It's responsible to push messages
* to the destination proc or relay broadcast messages. --Chao Mei
*
* Note: The completion handler could be executed on any cores within a node ???
* So in SMP mode when there's a comm thread, the completion handler should be carefully
* dealt with.
*
* Given lapi also provides an internal lapi thread to deal with network progress which
* will call this function (???), we should be careful with the following situations:
* 1) non SMP mode, with interrupt (lapi internal completion thread)
* 2) non SMP mode, with polling (machine layer is responsible for network progress)
* 3) SMP mode, no comm thread, with polling
* 4) SMP mode, no comm thread, with interrupt
* 5) SMP mode, with comm thread, with polling (not yet implemented, comm server is empty right now)
* 6) SMP mode, with comm thread, with interrupt??
*
* Currently, SMP mode without comm thread is undergoing implementation.
*
* This function is executed by LAPI internal threads. It seems that the number of internal
* completion handler threads could vary during the program. LAPI adaptively creates more
* threads if there are more outstanding messages!!!! This means pcqueue needs protection
* even in the nonsmp case!!!!
*
* --Chao Mei
*/
static void PumpMsgsComplete(lapi_handle_t *myLapiContext, void *am_info) {
int i;
char *msg = am_info;
int broot, destrank;
MACHSTATE3(2,"[%d] PumpMsgsComplete with msg %p (isImm=%d) begin {",CmiMyNode(), msg, CmiIsImmediate(msg));
#if ENSURE_MSG_PAIRORDER
MACHSTATE3(2,"msg %p info: srcpe=%d, seqno=%d", msg, CMI_MSG_SRCPE(msg), CMI_MSG_SEQNO(msg));
#endif
/**
* First, we check if the msg is a broadcast msg via spanning
* tree. If it is, it needs to call SendSpanningChildren to
* relay the broadcast, and then send the msg to every cores on
* this node.
*
* After the first check, we deal with normal messages.
* --Chao Mei
*/
/* It's the right place to relay the broadcast message */
/**
* 1. For in-order delivery, because this is the handler for
* receiving a message, and we assume the cross-network msgs are
* always delivered to the first proc (rank 0) of this node, we
* select the srcpe of the bcast msgs and the next msg seq no
* correspondingly.
*
* --Chao Mei
*/
#if ENSURE_MSG_PAIRORDER
broot = CMI_BROADCAST_ROOT(msg);
destrank = CMI_DEST_RANK(msg);
/* Only check proc-level msgs */
if (broot>=0
#if CMK_NODE_QUEUE_AVAILABLE
&& destrank != DGRAM_NODEMESSAGE
#endif
) {
MsgOrderInfo *info;
info = &CpvAccessOther(p2pMsgSeqInfo, destrank);
MACHSTATE1(2, "Check msg in-order for p2p msg %p", msg);
if (checkMsgInOrder(msg,info)) {
MACHSTATE(2,"} PumpMsgsComplete end ");
return;
}
}
#endif
handleOneRecvedMsg(CMI_MSG_SIZE(msg), msg);
MACHSTATE(2,"} PumpMsgsComplete end ");
return;
}
extern "C" void initQd(char **argv)
{
CpvInitialize(QdState*, _qd);
CpvAccess(_qd) = new QdState();
if (CmiMyRank() == 0) {
#if !defined(CMK_CPV_IS_SMP) && !CMK_SHARED_VARS_UNIPROCESSOR
CpvAccessOther(_qd, 1) = new QdState(); // for i/o interrupt
#endif
}
_qdHandlerIdx = CmiRegisterHandler((CmiHandler)_qdHandler);
_qdCommHandlerIdx = CmiRegisterHandler((CmiHandler)_qdCommHandler);
if (CmiGetArgIntDesc(argv,"+qd",&_dummy_dq, "QD time in seconds")) {
if (CmiMyPe()==0)
CmiPrintf("Charm++> Fake QD using %d seconds.\n", _dummy_dq);
}
}
