/* MSGQUEUE lock is not held */
void
MPIDI_Callback_process_trunc(pami_context_t context,
MPID_Request *rreq,
pami_recv_t *recv,
const void *sndbuf)
{
rreq->status.MPI_ERROR = MPI_ERR_TRUNCATE;
/* -------------------------------------------------------------- */
/* The data is already available, so we can just unpack it now. */
/* -------------------------------------------------------------- */
if (recv)
{
MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE);
rreq->mpid.uebuflen = MPIR_STATUS_GET_COUNT(rreq->status);
rreq->mpid.uebuf = MPIU_Malloc(MPIR_STATUS_GET_COUNT(rreq->status));
MPID_assert(rreq->mpid.uebuf != NULL);
rreq->mpid.uebuf_malloc = mpiuMalloc;
recv->addr = rreq->mpid.uebuf;
}
else
{
MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE);
rreq->mpid.uebuflen = MPIR_STATUS_GET_COUNT(rreq->status);
rreq->mpid.uebuf = (void*)sndbuf;
MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS);
MPID_Request_release(rreq);
}
}
/* MSGQUEUE lock is not held */
void
MPIDI_Callback_process_userdefined_dt(pami_context_t context,
const void * sndbuf,
size_t sndlen,
MPID_Request * rreq)
{
unsigned dt_contig, dt_size;
MPID_Datatype *dt_ptr;
MPI_Aint dt_true_lb;
MPIDI_Datatype_get_info(rreq->mpid.userbufcount,
rreq->mpid.datatype,
dt_contig,
dt_size,
dt_ptr,
dt_true_lb);
/* ----------------------------- */
/* Test for truncated message. */
/* ----------------------------- */
if (unlikely(sndlen > dt_size))
{
#if ASSERT_LEVEL > 0
MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf);
return;
#else
sndlen = dt_size;
#endif
}
/*
* This is to test that the fields don't need to be
* initialized. Remove after this doesn't fail for a while.
*/
if (likely (dt_contig))
{
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
void* rcvbuf = rreq->mpid.userbuf + dt_true_lb;;
memcpy(rcvbuf, sndbuf, sndlen);
MPIDI_Request_complete(rreq);
return;
}
MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE);
rreq->mpid.uebuflen = sndlen;
rreq->mpid.uebuf = (void*)sndbuf;
MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS);
MPID_Request_release(rreq);
}
static inline void
MPIDI_RecvShortCB(pami_context_t context,
const void * _msginfo,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
unsigned isSync)
{
MPID_assert(_msginfo != NULL);
const MPIDI_MsgInfo *msginfo = (const MPIDI_MsgInfo *)_msginfo;
MPID_Request * rreq = NULL;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, source, tag, context_id, msginfo->MPIseqno);
#endif
/* Match not found */
if (unlikely(rreq == NULL))
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request *newreq = MPIDI_Request_create2();
MPID_assert(newreq != NULL);
if (sndlen)
{
newreq->mpid.uebuflen = sndlen;
if (!TOKEN_FLOW_CONTROL_ON)
{
newreq->mpid.uebuf = MPL_malloc(sndlen);
newreq->mpid.uebuf_malloc = mpiuMalloc;
}
else
{
#if TOKEN_FLOW_CONTROL
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
newreq->mpid.uebuf = MPIDI_mm_alloc(sndlen);
newreq->mpid.uebuf_malloc = mpidiBufMM;
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#else
MPID_assert_always(0);
#endif
}
MPID_assert(newreq->mpid.uebuf != NULL);
}
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, PAMIX_Endpoint_query(sender), tag, context_id, msginfo->MPIseqno);
#endif
if (unlikely(rreq == NULL))
{
MPIDI_Callback_process_unexp(newreq, context, msginfo, sndlen, sender, sndbuf, NULL, isSync);
/* request is always complete now */
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Token_cntr[source].unmatched++;
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_release(newreq);
goto fn_exit_short;
}
else
{
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_discard(newreq);
}
}
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Update_rettoks(source);
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* the receive queue processing has been completed and we found match*/
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, sndlen);
MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE);
MPIDI_Request_cpyPeerRequestH(rreq, msginfo);
MPIDI_Request_setSync (rreq, isSync);
MPIDI_Request_setRzv (rreq, 0);
/* ----------------------------- */
/* Request was already posted. */
/* ----------------------------- */
if (unlikely(isSync))
MPIDI_SyncAck_post(context, rreq, PAMIX_Endpoint_query(sender));
if (unlikely(HANDLE_GET_KIND(rreq->mpid.datatype) != HANDLE_KIND_BUILTIN))
{
MPIDI_Callback_process_userdefined_dt(context, sndbuf, sndlen, rreq);
goto fn_exit_short;
}
size_t dt_size = rreq->mpid.userbufcount * MPID_Datatype_get_basic_size(rreq->mpid.datatype);
/* ----------------------------- */
/* Test for truncated message. */
/* ----------------------------- */
if (unlikely(sndlen > dt_size))
{
#if ASSERT_LEVEL > 0
MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf);
goto fn_exit_short;
#else
sndlen = dt_size;
#endif
}
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
void* rcvbuf = rreq->mpid.userbuf;
if (sndlen > 0)
{
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf))
{
cudaError_t cudaerr = CudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice);
}
else
#endif
memcpy(rcvbuf, sndbuf, sndlen);
}
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
MPIDI_Request_complete(rreq);
fn_exit_short:
#ifdef OUT_OF_ORDER_HANDLING
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs>0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#endif
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
/* MSGQUEUE lock must be held by caller */
void
MPIDI_Callback_process_unexp(MPID_Request *newreq,
pami_context_t context,
const MPIDI_MsgInfo * msginfo,
size_t sndlen,
pami_endpoint_t sender,
const void * sndbuf,
pami_recv_t * recv,
unsigned isSync)
{
MPID_Request *rreq = NULL;
/* ---------------------------------------------------- */
/* Fallback position: */
/* + Request was not posted, or */
/* + Request was long & not contiguous. */
/* We must allocate enough space to hold the message. */
/* The temporary buffer will be unpacked later. */
/* ---------------------------------------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_AEU(newreq, rank, tag, context_id);
#else
unsigned msg_seqno = msginfo->MPIseqno;
rreq = MPIDI_Recvq_AEU(newreq, rank, PAMIX_Endpoint_query(sender), tag, context_id, msg_seqno);
#endif
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, sndlen);
MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE);
MPIDI_Request_cpyPeerRequestH(rreq, msginfo);
MPIDI_Request_setSync (rreq, isSync);
/* Set the rank of the sender if a sync msg. */
#ifndef OUT_OF_ORDER_HANDLING
if (isSync)
{
#endif
MPIDI_Request_setPeerRank_comm(rreq, rank);
MPIDI_Request_setPeerRank_pami(rreq, PAMIX_Endpoint_query(sender));
#ifndef OUT_OF_ORDER_HANDLING
}
#endif
MPID_assert(!sndlen || rreq->mpid.uebuf != NULL);
TRACE_MEMSET_R(PAMIX_Endpoint_query(sender),msg_seqno,recv_status);
TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),msgid,msginfo->MPIseqno);
TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rtag,tag);
TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rctx,msginfo->MPIctxt);
TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rlen,sndlen);
TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),fl.f.sync,isSync);
TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rsource,PAMIX_Endpoint_query(sender));
TRACE_SET_REQ_VAL(rreq->mpid.idx,(msginfo->MPIseqno & SEQMASK));
if (recv != NULL)
{
recv->local_fn = MPIDI_RecvDoneCB_mutexed;
recv->cookie = rreq;
/* -------------------------------------------------- */
/* Let PAMI know where to put the rest of the data. */
/* -------------------------------------------------- */
recv->addr = rreq->mpid.uebuf;
}
else
{
/* ------------------------------------------------- */
/* We have the data; copy it and complete the msg. */
/* ------------------------------------------------- */
memcpy(rreq->mpid.uebuf, sndbuf, sndlen);
MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS);
/* caller must release rreq, after unlocking MSGQUEUE */
}
}
void
MPIDI_RecvMsg_Unexp(MPID_Request * rreq,
void * buf,
int count,
MPI_Datatype datatype)
{
/* ------------------------------------------------------------ */
/* message was found in unexpected queue */
/* ------------------------------------------------------------ */
/* We must acknowledge synchronous send requests */
/* The recvnew callback will acknowledge the posted messages */
/* Recv functions will ack the messages that are unexpected */
/* ------------------------------------------------------------ */
#ifdef MPIDI_TRACE
MPIDI_In_cntr[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ=1;
#endif
if (MPIDI_Request_isRzv(rreq))
{
const unsigned is_sync = MPIDI_Request_isSync(rreq);
const unsigned is_zero = (rreq->mpid.envelope.length==0);
/* -------------------------------------------------------- */
/* Received an expected flow-control rendezvous RTS. */
/* This is very similar to the found/incomplete case */
/* -------------------------------------------------------- */
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN)
{
MPID_Datatype_get_ptr(datatype, rreq->mpid.datatype_ptr);
MPID_Datatype_add_ref(rreq->mpid.datatype_ptr);
}
if (likely((is_sync+is_zero) == 0))
MPIDI_Context_post(MPIDI_Context_local(rreq), &rreq->mpid.post_request, MPIDI_RendezvousTransfer, rreq);
else if (is_sync != 0)
MPIDI_Context_post(MPIDI_Context_local(rreq), &rreq->mpid.post_request, MPIDI_RendezvousTransfer_SyncAck, rreq);
else
MPIDI_Context_post(MPIDI_Context_local(rreq), &rreq->mpid.post_request, MPIDI_RendezvousTransfer_zerobyte, rreq);
}
else
{
if (MPID_cc_is_complete(&rreq->cc))
{
if (unlikely(MPIDI_Request_isSync(rreq)))
{
/* Post this to the context for asynchronous progresss. We cannot do
* the send-immediate inline here because we may not have the
* context locked (its is being asynchrously advanced).
* Must "uncomplete" the message (increment the ref and completion counts) so we
* hold onto this request object until this send has completed. When MPIDI_SyncAck_handoff
* finishes sending the ack, it will complete the request, decrementing the ref and
* completion counts.
*/
MPIDI_Request_uncomplete(rreq);
MPIDI_Send_post(MPIDI_SyncAck_handoff, rreq);
}
/* -------------------------------- */
/* request is complete */
/* -------------------------------- */
if (rreq->mpid.uebuf != NULL)
{
if (likely(rreq->status.cancelled == FALSE))
{
MPIDI_msg_sz_t _count=0;
MPIDI_Buffer_copy(rreq->mpid.uebuf,
rreq->mpid.uebuflen,
MPI_CHAR,
&rreq->status.MPI_ERROR,
buf,
count,
datatype,
&_count,
&rreq->status.MPI_ERROR);
rreq->status.count = _count;
}
}
else
{
MPID_assert(rreq->mpid.uebuflen == 0);
rreq->status.count = 0;
}
}
else
{
/* -------------------------------- */
/* request is incomplete */
/* -------------------------------- */
if (unlikely(MPIDI_Request_isSync(rreq)))
{
/* Post this to the context for asynchronous progresss. We cannot do
* the send-immediate inline here because we may not have the
* context locked (its is being asynchrously advanced).
* Must "uncomplete" the message (increment the ref and completion counts) so we
* hold onto this request object until this send has completed. When MPIDI_SyncAck_handoff
* finishes sending the ack, it will complete the request, decrementing the ref and
* completion counts.
*/
MPIDI_Request_uncomplete(rreq);
MPIDI_Send_post(MPIDI_SyncAck_handoff, rreq);
}
if(rreq->status.cancelled == FALSE)
{
MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE);
}
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN)
{
MPID_Datatype_get_ptr(datatype, rreq->mpid.datatype_ptr);
MPID_Datatype_add_ref(rreq->mpid.datatype_ptr);
}
}
}
}
