static inline
int ADD_SUFFIX(MPID_nem_ofi_recv_callback)(cq_tagged_entry_t * wc, MPID_Request * rreq)
{
int err0, err1, src, mpi_errno = MPI_SUCCESS;
uint64_t ssend_bits;
MPIDI_msg_sz_t sz;
MPIDI_VC_t *vc;
MPID_Request *sync_req;
BEGIN_FUNC(FCNAME);
/* ---------------------------------------------------- */
/* Populate the MPI Status and unpack noncontig buffer */
/* ---------------------------------------------------- */
rreq->status.MPI_ERROR = MPI_SUCCESS;
#if API_SET == API_SET_1
rreq->status.MPI_SOURCE = get_source(wc->tag);
#elif API_SET == API_SET_2
rreq->status.MPI_SOURCE = wc->data;
#endif
src = rreq->status.MPI_SOURCE;
rreq->status.MPI_TAG = get_tag(wc->tag);
REQ_OFI(rreq)->req_started = 1;
MPIR_STATUS_SET_COUNT(rreq->status, wc->len);
if (REQ_OFI(rreq)->pack_buffer) {
MPIDI_CH3U_Buffer_copy(REQ_OFI(rreq)->pack_buffer,
MPIR_STATUS_GET_COUNT(rreq->status),
MPI_BYTE, &err0, rreq->dev.user_buf,
rreq->dev.user_count, rreq->dev.datatype, &sz, &err1);
MPIR_STATUS_SET_COUNT(rreq->status, sz);
MPIU_Free(REQ_OFI(rreq)->pack_buffer);
if (err0 || err1) {
rreq->status.MPI_ERROR = MPI_ERR_TYPE;
}
}
if ((wc->tag & MPID_PROTOCOL_MASK) == MPID_SYNC_SEND) {
/* ---------------------------------------------------- */
/* Ack the sync send and wait for the send request */
/* completion(when callback executed. A protocol bit */
/* MPID_SYNC_SEND_ACK is set in the tag bits to provide */
/* separation of MPI messages and protocol messages */
/* ---------------------------------------------------- */
vc = REQ_OFI(rreq)->vc;
if (!vc) { /* MPI_ANY_SOURCE -- Post message from status, complete the VC */
vc = rreq->comm->dev.vcrt->vcr_table[src];
MPIU_Assert(vc);
}
#if API_SET == API_SET_1
ssend_bits = init_sendtag(rreq->dev.match.parts.context_id,
rreq->comm->rank, rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK);
#elif API_SET == API_SET_2
ssend_bits = init_sendtag_2(rreq->dev.match.parts.context_id,
rreq->status.MPI_TAG, MPID_SYNC_SEND_ACK);
#endif
MPID_nem_ofi_create_req(&sync_req, 1);
sync_req->dev.OnDataAvail = NULL;
sync_req->dev.next = NULL;
REQ_OFI(sync_req)->event_callback = MPID_nem_ofi_sync_recv_callback;
REQ_OFI(sync_req)->parent = rreq;
#if API_SET == API_SET_1
FI_RC_RETRY(fi_tsend(gl_data.endpoint,
#elif API_SET == API_SET_2
FI_RC_RETRY(fi_tsenddata(gl_data.endpoint,
#endif
NULL,
0,
gl_data.mr,
#if API_SET == API_SET_2
rreq->comm->rank,
#endif
VC_OFI(vc)->direct_addr,
ssend_bits, &(REQ_OFI(sync_req)->ofi_context)), tsend);
}
int MPIDI_Isend_self(const void * buf, int count, MPI_Datatype datatype, int rank, int tag, MPID_Comm * comm, int context_offset,
int type, MPID_Request ** request)
{
MPIDI_Message_match match;
MPID_Request * sreq;
MPID_Request * rreq;
MPIDI_VC_t * vc;
#if defined(MPID_USE_SEQUENCE_NUMBERS)
MPID_Seqnum_t seqnum;
#endif
int found;
int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"sending message to self");
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIDI_Request_set_type(sreq, type);
MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_SELF_MSG);
match.parts.rank = rank;
match.parts.tag = tag;
match.parts.context_id = comm->context_id + context_offset;
MPIU_THREAD_CS_ENTER(MSGQUEUE,);
rreq = MPIDI_CH3U_Recvq_FDP_or_AEU(&match, &found);
/* --BEGIN ERROR HANDLING-- */
if (rreq == NULL)
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
sreq = NULL;
MPIU_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**nomem",
"**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
goto fn_exit;
}
/* --END ERROR HANDLING-- */
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPIDI_Request_set_seqnum(rreq, seqnum);
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
if (found)
{
MPIDI_msg_sz_t data_sz;
/* we found a posted req, which we now own, so we can release the CS */
MPIU_THREAD_CS_EXIT(MSGQUEUE,);
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,
"found posted receive request; copying data");
MPIDI_CH3U_Buffer_copy(buf, count, datatype, &sreq->status.MPI_ERROR,
rreq->dev.user_buf, rreq->dev.user_count, rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
rreq->status.count = (int)data_sz;
MPID_REQUEST_SET_COMPLETED(rreq);
MPID_Request_release(rreq);
/* sreq has never been seen by the user or outside this thread, so it is safe to reset ref_count and cc */
MPIU_Object_set_ref(sreq, 1);
MPID_cc_set(&sreq->cc, 0);
}
else
{
if (type != MPIDI_REQUEST_TYPE_RSEND)
int MPIDI_CH3_RecvFromSelf( MPID_Request *rreq, void *buf, MPI_Aint count,
MPI_Datatype datatype )
#endif
{
MPID_Request * const sreq = rreq->partner_request;
int mpi_errno = MPI_SUCCESS;
if (sreq != NULL)
{
MPIDI_msg_sz_t data_sz;
#if defined(FINEGRAIN_MPI)
/* FG: Zerocopy */
void * buf = (void *) (*buf_handle);
if ( MPIDI_Request_get_self_zerocopy_flag(sreq) && MPIDI_Request_get_self_zerocopy_flag(rreq) )
{
int rdt_contig;
MPI_Aint rdt_true_lb;
MPID_Datatype * rdt_ptr;
/* Unexpected Send-Collocated MPIX_Zsend/Izsend - MPIX_Zrecv/Izrecv pairing */
MPIU_Assert(NULL == rreq->dev.user_buf);
*(rreq->dev.user_buf_handle) = (void*) (*(sreq->dev.user_buf_handle));
MPIDI_Datatype_get_info(count, datatype, rdt_contig, data_sz, rdt_ptr, rdt_true_lb);
/* MPIX_Zsend buf_handle can't be set to NULL as we don't have
a ptr to void **. */
}
else if( MPIDI_Request_get_self_zerocopy_flag(sreq) && !MPIDI_Request_get_self_zerocopy_flag(rreq) ){
/* Unexpected Send-Collocated MPIX_Zsend/Izsend<=>MPI_Recv/Irecv pairing. Freeing sender buffer */
MPIDI_CH3U_Buffer_copy(*(sreq->dev.user_buf_handle), sreq->dev.user_count,
sreq->dev.datatype, &sreq->status.MPI_ERROR,
buf, count, datatype, &data_sz,
&rreq->status.MPI_ERROR);
/* Free the sender's buffer */
MPIU_Free(*(sreq->dev.user_buf_handle));
}
else if( !MPIDI_Request_get_self_zerocopy_flag(sreq) && MPIDI_Request_get_self_zerocopy_flag(rreq) ){
/* Unexpected Send-Collocated MPI_Send/Isend - MPIX_Zrecv/Izrecv pairing. Allocating receiver's buffer. */
MPIU_Assert(NULL == rreq->dev.user_buf);
/* Added checks for buffer count size as is done in MPIDI_CH3U_Buffer_copy() */
MPIDI_CH3U_Buffer_allocate(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, &sreq->status.MPI_ERROR,
rreq->dev.user_buf_handle, rreq->dev.user_count,
rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
MPIDI_CH3U_Buffer_copy(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, &sreq->status.MPI_ERROR,
*(rreq->dev.user_buf_handle), rreq->dev.user_count,
rreq->dev.datatype, &data_sz, &rreq->status.MPI_ERROR);
} else {
/* Unexpected Send-Collocated MPI_Send/Isend - MPI_Recv/Irecv pairing */
#endif /* matches #if defined(FINEGRAIN_MPI) */
MPIDI_CH3U_Buffer_copy(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, &sreq->status.MPI_ERROR,
buf, count, datatype, &data_sz,
&rreq->status.MPI_ERROR);
#if defined(FINEGRAIN_MPI)
}
#endif
MPIR_STATUS_SET_COUNT(rreq->status, data_sz);
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else
{
/* The sreq is missing which means an error occurred.
rreq->status.MPI_ERROR should have been set when the
error was detected. */
}
/* no other thread can possibly be waiting on rreq, so it is safe to
reset ref_count and cc */
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
