/**
* \brief The standard callback for a new short message
* \note Because this is a short message, the data is already received
* \param[in] clientdata Unused
* \param[in] msginfo The 16-byte msginfo struct
* \param[in] count The number of msginfo quads (1)
* \param[in] senderrank The sender's rank
* \param[in] sndlen The length of the incoming data
* \param[in] sndbuf Where the data is stored
*/
void MPIDI_BG2S_RecvShortCB(void * clientdata,
const DCQuad * msgquad,
unsigned count,
size_t senderrank,
const char * sndbuf,
size_t sndlen)
{
const MPIDI_DCMF_MsgInfo *msginfo = (const MPIDI_DCMF_MsgInfo *)msgquad;
MPID_Request * rreq = NULL;
int found;
int rcvlen = sndlen;
/* Handle cancel requests */
if (msginfo->msginfo.type == MPIDI_DCMF_REQUEST_TYPE_CANCEL_REQUEST)
{
MPIDI_DCMF_procCancelReq(msginfo, senderrank);
return;
}
/* -------------------------- */
/* match request */
/* -------------------------- */
MPIDI_Message_match match;
match.rank = msginfo->msginfo.MPIrank;
match.tag = msginfo->msginfo.MPItag;
match.context_id = msginfo->msginfo.MPIctxt;
rreq = MPIDI_Recvq_FDP_or_AEU(match.rank, match.tag, match.context_id, &found);
if (rreq == NULL)
{
/* ------------------------------------------------- */
/* we have failed to match the request. */
/* allocate and initialize a request object instead. */
/* ------------------------------------------------- */
int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_FATAL,
"mpid_recv",
__LINE__,
MPI_ERR_OTHER,
"**nomem", 0);
rreq->status.MPI_ERROR = mpi_errno;
rreq->status.count = 0;
MPID_Abort(NULL, mpi_errno, -1, "Cannot allocate message");
}
/* -------------------------------------- */
/* Signal that the recv has been started. */
/* -------------------------------------- */
MPID_Progress_signal ();
/* ------------------------ */
/* copy in information */
/* ------------------------ */
rreq->status.MPI_SOURCE = match.rank;
rreq->status.MPI_TAG = match.tag;
MPID_Request_setPeerRank(rreq,senderrank);
MPID_Request_setPeerRequest(rreq,msginfo->msginfo.req);
MPID_Request_setSync(rreq, msginfo->msginfo.isSync);
MPID_Request_setRzv(rreq, 0);
/*
*
* Whitespace to sync lines of code with mpidi_callback.c
*
*/
/* ----------------------------------------- */
/* figure out target buffer for request data */
/* ----------------------------------------- */
MPID_Request_setCA(rreq, MPIDI_DCMF_CA_COMPLETE);
rreq->status.count = rcvlen;
if (found)
{
/* --------------------------- */
/* request was already posted. */
/* if synchronized, post ack. */
/* --------------------------- */
if (msginfo->msginfo.isSync)
MPIDI_DCMF_postSyncAck(rreq);
/* -------------------------------------- */
/* calculate message length for reception */
/* calculate receive message "count" */
/* -------------------------------------- */
unsigned dt_contig, dt_size;
MPID_Datatype *dt_ptr;
MPI_Aint dt_true_lb;
MPIDI_Datatype_get_info (rreq->dcmf.userbufcount,
rreq->dcmf.datatype,
dt_contig,
dt_size,
dt_ptr,
dt_true_lb);
/* -------------------------------------- */
/* test for truncated message. */
/* -------------------------------------- */
if (rcvlen > dt_size)
{
rcvlen = dt_size;
rreq->status.MPI_ERROR = MPI_ERR_TRUNCATE;
rreq->status.count = rcvlen;
}
/* -------------------------------------- */
/* if buffer is contiguous ... */
/* -------------------------------------- */
if (dt_contig)
{
char *rcvbuf;
rreq->dcmf.uebuf = NULL;
rreq->dcmf.uebuflen = 0;
rcvbuf = (char *)rreq->dcmf.userbuf + dt_true_lb;
memcpy(rcvbuf, sndbuf, rcvlen);
MPIDI_DCMF_RecvDoneCB(rreq, NULL);
return;
}
/* --------------------------------------------- */
/* buffer is non-contiguous. we need to specify */
/* the send buffer as temporary and unpack. */
/* --------------------------------------------- */
else
{
MPID_Request_setCA(rreq, MPIDI_DCMF_CA_UNPACK_UEBUF_AND_COMPLETE_NOFREE);
rreq->dcmf.uebuflen = rcvlen ;
rreq->dcmf.uebuf = (char *) sndbuf ;
MPIDI_DCMF_RecvDoneCB(rreq, NULL);
return;
}
}
/* ------------------------------------------------------------- */
/* Request was not posted. We must allocate enough space to hold */
/* the message temporarily and copy the data into the temporary */
/* buffer. The temporary buffer will be unpacked later. */
/* ------------------------------------------------------------- */
rreq->dcmf.uebuflen = rcvlen ;
if ((rreq->dcmf.uebuf = MPIU_Malloc (rcvlen)) == NULL)
{
/* ------------------------------------ */
/* creation of temporary buffer failed. */
/* we are in trouble and must bail out. */
/* ------------------------------------ */
int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_FATAL,
"mpid_recv",
__LINE__,
MPI_ERR_OTHER,
"**nomem", 0);
rreq->status.MPI_ERROR = mpi_errno;
rreq->status.count = 0;
MPID_Abort(NULL, mpi_errno, -1, "Cannot allocate unexpected buffer");
}
/* ------------------------------------------------ */
/* Copy the data into the unexpected buffer. */
/* ------------------------------------------------ */
memcpy(rreq->dcmf.uebuf, sndbuf, rreq->dcmf.uebuflen);
MPIDI_DCMF_RecvDoneCB(rreq, NULL);
return;
}
/**
* \brief The callback for a new RZV RTS
* \note Because this is a short message, the data is already received
* \param[in] context The context on which the message is being received.
* \param[in] sender The origin endpoint
* \param[in] _msginfo The extended header information
* \param[in] msginfo_size The size of the extended header information
* \param[in] is_zero_byte The rendezvous message is zero bytes in length.
*/
void
MPIDI_RecvRzvCB_impl(pami_context_t context,
pami_endpoint_t sender,
const void * _msginfo,
size_t msginfo_size,
const unsigned is_zero_byte)
{
MPID_assert(_msginfo != NULL);
MPID_assert(msginfo_size == sizeof(MPIDI_MsgEnvelope));
const MPIDI_MsgEnvelope * envelope = (const MPIDI_MsgEnvelope *)_msginfo;
const MPIDI_MsgInfo * msginfo = (const MPIDI_MsgInfo *)&envelope->msginfo;
MPID_Request * rreq = NULL;
int found;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPID_Request *newreq = MPIDI_Request_create2();
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, tag, context_id, &found);
#else
rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, source, tag, context_id, msginfo->MPIseqno, &found);
#endif
TRACE_ERR("RZV CB for req=%p remote-mr=0x%llx bytes=%zu (%sfound)\n",
rreq,
*(unsigned long long*)&envelope->envelope.memregion,
envelope->envelope.length,
found?"":"not ");
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, envelope->length);
MPIDI_Request_setPeerRank_comm(rreq, rank);
MPIDI_Request_setPeerRank_pami(rreq, source);
MPIDI_Request_cpyPeerRequestH (rreq, msginfo);
MPIDI_Request_setSync (rreq, msginfo->isSync);
MPIDI_Request_setRzv (rreq, 1);
/* ----------------------------------------------------- */
/* Save the rendezvous information for when the target */
/* node calls a receive function and the data is */
/* retreived from the origin node. */
/* ----------------------------------------------------- */
if (is_zero_byte)
{
rreq->mpid.envelope.length = 0;
rreq->mpid.envelope.data = NULL;
}
else
{
#ifdef USE_PAMI_RDMA
memcpy(&rreq->mpid.envelope.memregion,
&envelope->memregion,
sizeof(pami_memregion_t));
#else
rreq->mpid.envelope.memregion_used = envelope->memregion_used;
if(envelope->memregion_used)
{
memcpy(&rreq->mpid.envelope.memregion,
&envelope->memregion,
sizeof(pami_memregion_t));
}
rreq->mpid.envelope.data = envelope->data;
#endif
rreq->mpid.envelope.length = envelope->length;
TRACE_SET_R_VAL(source,(rreq->mpid.idx),req,rreq);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,envelope->length);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),fl.f.sync,msginfo->isSync);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.rzv);
if (TOKEN_FLOW_CONTROL_ON)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
}
/* ----------------------------------------- */
/* figure out target buffer for request data */
/* ----------------------------------------- */
if (found)
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
/* --------------------------- */
/* if synchronized, post ack. */
/* --------------------------- */
if (unlikely(MPIDI_Request_isSync(rreq)))
MPIDI_SyncAck_post(context, rreq, MPIDI_Request_getPeerRank_pami(rreq));
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
if (is_zero_byte)
MPIDI_RecvRzvDoneCB_zerobyte(context, rreq, PAMI_SUCCESS);
else
{
MPIDI_RendezvousTransfer(context, rreq);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.sync_com_in_HH);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.matchedInHH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
}
MPID_Request_discard(newreq);
}
/* ------------------------------------------------------------- */
/* Request was not posted. */
/* ------------------------------------------------------------- */
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
/*
* This is to test that the fields don't need to be
* initialized. Remove after this doesn't fail for a while.
*/
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
/* rreq->mpid.uebuf = NULL; */
/* rreq->mpid.uebuflen = 0; */
#ifdef OUT_OF_ORDER_HANDLING
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs > 0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
