static void _mxm_recv_completion_cb(void *context)
{
MPID_Request *req = (MPID_Request *) context;
mxm_recv_req_t *mxm_rreq;
MPID_nem_mxm_req_area *req_area = NULL;
MPIU_Assert(req);
_dbg_mxm_out_req(req);
req_area = REQ_BASE(req);
_mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status);
mxm_rreq = &req_area->mxm_req->item.recv;
req->status.MPI_TAG = _mxm_tag_mxm2mpi(mxm_rreq->completion.sender_tag);
req->status.MPI_SOURCE = mxm_rreq->completion.sender_imm;
req->dev.recv_data_sz = mxm_rreq->completion.actual_len;
MPIR_STATUS_SET_COUNT(req->status, req->dev.recv_data_sz);
if (req->ch.vc) {
MPID_nem_mxm_vc_area *vc_area = VC_BASE(req->ch.vc);
list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue);
}
else {
list_enqueue(&mxm_obj->free_queue, &req_area->mxm_req->queue);
}
_dbg_mxm_output(5, "========> %s RECV req %p status %d\n",
(MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"),
req, req->status.MPI_ERROR);
if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) {
_mxm_handle_rreq(req);
}
}
int MPID_nem_mxm_anysource_matched(MPID_Request * req)
{
mxm_error_t ret = MXM_OK;
MPID_nem_mxm_req_area *req_area = NULL;
int matched = FALSE;
/* This function is called when an anysource request in the posted
* receive queue is matched and dequeued see MPIDI_POSTED_RECV_DEQUEUE_HOOK().
* It returns 0(FALSE) if the req was not matched by mxm and non-zero(TRUE)
* otherwise.
* This happens
* when the channel supports shared-memory and network communication
* with a network capable of matching, and the same request is matched
* by the network and, e.g., shared-memory.
*/
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_MATCHED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_MATCHED);
_dbg_mxm_output(5, "Any Source ========> Matching req %p \n", req);
req_area = REQ_BASE(req);
ret = mxm_req_cancel_recv(&req_area->mxm_req->item.recv);
mxm_req_wait(&req_area->mxm_req->item.base);
if (req_area->mxm_req->item.base.error != MXM_ERR_CANCELED) {
matched = TRUE;
}
_dbg_mxm_out_req(req);
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_MATCHED);
return matched;
}
void MPID_nem_mxm_anysource_posted(MPID_Request * req)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_POSTED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_POSTED);
_dbg_mxm_output(5, "Any Source ========> Posting req %p \n", req);
mpi_errno = MPID_nem_mxm_recv(NULL, req);
MPIU_Assert(mpi_errno == MPI_SUCCESS);
_dbg_mxm_out_req(req);
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ANYSOURCE_POSTED);
}
static void _mxm_send_completion_cb(void *context)
{
MPID_Request *req = (MPID_Request *) context;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIU_Assert(req);
_dbg_mxm_out_req(req);
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status);
list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue);
_dbg_mxm_output(5, "========> %s SEND req %p status %d\n",
(MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"),
req, req->status.MPI_ERROR);
if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) {
_mxm_handle_sreq(req);
}
}
int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype datatype,
int rank, int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** sreq_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq = NULL;
MPID_Datatype *dt_ptr;
int dt_contig;
MPIDI_msg_sz_t data_sz;
MPI_Aint dt_true_lb;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISSEND);
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
/* create a request */
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIU_Assert(sreq != NULL);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPID_Datatype_get_ptr(datatype, sreq->dev.datatype_ptr);
MPID_Datatype_add_ref(sreq->dev.datatype_ptr);
}
sreq->partner_request = NULL;
sreq->dev.OnDataAvail = NULL;
sreq->dev.tmpbuf = NULL;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"isSend ========> Sending USER msg for req %p (context %d to %d tag %d size %d) \n",
sreq, comm->context_id + context_offset, rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area-> ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (data_sz) {
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
MPIDI_msg_sz_t last;
MPI_Aint packsize = 0;
sreq->ch.noncontig = TRUE;
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER,
"**nomem", "**nomem %s", "MPID_Segment_alloc");
MPIR_Pack_size_impl(count, datatype, &packsize);
last = data_sz;
if (packsize > 0) {
sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
MPIU_Assert(sreq->dev.tmpbuf);
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = sreq->dev.tmpbuf;
req_area->iov_buf[0].length = last;
}
}
}
vc_area->pending_sends += 1;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC,
(mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag,
comm->context_id
+
context_offset),
0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
_dbg_mxm_out_req(sreq);
fn_exit:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISSEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_RECV);
MPIU_Assert(rreq);
MPIU_Assert(((rreq->dev.match.parts.rank == MPI_ANY_SOURCE) && (vc == NULL)) ||
(vc && !vc->ch.is_local));
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz,
dt_ptr, dt_true_lb);
{
MPIU_Context_id_t context_id = rreq->dev.match.parts.context_id;
int tag = rreq->dev.match.parts.tag;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
mxm_mq_h *mq_h_v = (mxm_mq_h *) rreq->comm->dev.ch.netmod_priv;
rreq->dev.OnDataAvail = NULL;
rreq->dev.tmpbuf = NULL;
rreq->ch.vc = vc;
rreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"Recv ========> Getting USER msg for req %p (context %d from %d tag %d size %d) \n",
rreq, context_id, rreq->dev.match.parts.rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(rreq);
req_area->ctx = rreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (rreq->dev.user_buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
rreq->ch.noncontig = TRUE;
mpi_errno = _mxm_process_rdtype(&rreq, rreq->dev.datatype, dt_ptr, data_sz,
rreq->dev.user_buf, rreq->dev.user_count,
&req_area->iov_buf, &req_area->iov_count);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
mpi_errno = _mxm_irecv((vc_area ? vc_area->mxm_ep : NULL), req_area,
tag,
(rreq->comm ? mq_h_v[0] : mxm_obj->mxm_mq), _mxm_tag_mpi2mxm(tag,
context_id));
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (vc)
_dbg_mxm_out_req(rreq);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_RECV);
return mpi_errno;
fn_fail:ATTRIBUTE((unused))
goto fn_exit;
}
