int MPID_nem_mxm_vc_terminate(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc);
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MXM_VC_TERMINATE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MXM_VC_TERMINATE);
if (vc->state != MPIDI_VC_STATE_CLOSED) {
/* VC is terminated as a result of a fault. Complete
* outstanding sends with an error and terminate connection
* immediately. */
MPIR_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank);
}
else {
while (vc_area->pending_sends > 0)
MPID_nem_mxm_poll(FALSE);
}
mpi_errno = MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MXM_VC_TERMINATE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int _mxm_handle_sreq(MPID_Request * req)
{
int complete = FALSE;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length));
vc_area->pending_sends -= 1;
if (req->dev.tmpbuf) {
if (req->dev.datatype_ptr || req->ch.noncontig) {
MPIU_Free(req->dev.tmpbuf);
}
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
MPIU_Free(req_area->iov_buf);
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
MPIDI_CH3U_Handle_send_req(req->ch.vc, req, &complete);
MPIU_Assert(complete == TRUE);
return complete;
}
int MPID_nem_mxm_vc_destroy(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MXM_VC_DESTROY);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MXM_VC_DESTROY);
/* Do nothing because
* finalize is called before vc destroy as result it is not possible
* to destroy endpoint here
*/
#if 0
MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc);
if (vc_area->ctx == vc) {
mpi_errno = _mxm_disconnect(vc_area->mxm_ep);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
#endif
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MXM_VC_DESTROY);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static void _mxm_recv_completion_cb(void *context)
{
MPID_Request *req = (MPID_Request *) context;
mxm_recv_req_t *mxm_rreq;
MPID_nem_mxm_req_area *req_area = NULL;
MPIU_Assert(req);
_dbg_mxm_out_req(req);
req_area = REQ_BASE(req);
_mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status);
mxm_rreq = &req_area->mxm_req->item.recv;
req->status.MPI_TAG = _mxm_tag_mxm2mpi(mxm_rreq->completion.sender_tag);
req->status.MPI_SOURCE = mxm_rreq->completion.sender_imm;
req->dev.recv_data_sz = mxm_rreq->completion.actual_len;
MPIR_STATUS_SET_COUNT(req->status, req->dev.recv_data_sz);
if (req->ch.vc) {
MPID_nem_mxm_vc_area *vc_area = VC_BASE(req->ch.vc);
list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue);
}
else {
list_enqueue(&mxm_obj->free_queue, &req_area->mxm_req->queue);
}
_dbg_mxm_output(5, "========> %s RECV req %p status %d\n",
(MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"),
req, req->status.MPI_ERROR);
if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) {
_mxm_handle_rreq(req);
}
}
int MPID_nem_mxm_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data,
MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq = NULL;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mxm_iStartContigMsg");
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);
/* create a request */
sreq = MPID_Request_create();
MPIU_Assert(sreq != NULL);
MPIU_Object_set_ref(sreq, 2);
MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t));
sreq->kind = MPID_REQUEST_SEND;
sreq->dev.OnDataAvail = NULL;
sreq->dev.tmpbuf = NULL;
_dbg_mxm_output(5,
"iStartContigMsg ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n",
vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t),
data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area->ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt);
req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t);
if (data_sz) {
req_area->iov_count = 2;
req_area->iov_buf[1].ptr = (void *) data;
req_area->iov_buf[1].length = data_sz;
}
vc_area->pending_sends += 1;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM,
mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
fn_exit:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_mxm_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz,
void *data, MPIDI_msg_sz_t data_sz)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mxm_iSendContig");
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);
MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t));
_dbg_mxm_output(5,
"iSendContig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n",
vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t),
data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area->ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[req_area->iov_count].ptr = (void *) &(sreq->dev.pending_pkt);
req_area->iov_buf[req_area->iov_count].length = sizeof(MPIDI_CH3_Pkt_t);
(req_area->iov_count)++;
if (sreq->dev.ext_hdr_sz != 0) {
req_area->iov_buf[req_area->iov_count].ptr = (void *) (sreq->dev.ext_hdr_ptr);
req_area->iov_buf[req_area->iov_count].length = sreq->dev.ext_hdr_sz;
(req_area->iov_count)++;
}
if (data_sz) {
req_area->iov_buf[req_area->iov_count].ptr = (void *) data;
req_area->iov_buf[req_area->iov_count].length = data_sz;
(req_area->iov_count)++;
}
vc_area->pending_sends += 1;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM,
mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
MPIDI_msg_sz_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t last;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "MPID_nem_mxm_iSendNoncontig");
MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t));
_dbg_mxm_output(5,
"SendNoncontig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n",
vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t),
sreq->dev.segment_size);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area->ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt);
req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t);
MPIU_Assert(sreq->dev.segment_first == 0);
last = sreq->dev.segment_size;
if (last > 0) {
sreq->dev.tmpbuf = MPIU_Malloc((size_t) sreq->dev.segment_size);
MPIU_Assert(sreq->dev.tmpbuf);
MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.tmpbuf);
MPIU_Assert(last == sreq->dev.segment_size);
req_area->iov_count = 2;
req_area->iov_buf[1].ptr = sreq->dev.tmpbuf;
req_area->iov_buf[1].length = last;
}
vc_area->pending_sends += 1;
sreq->ch.vc = vc;
sreq->ch.noncontig = TRUE;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM,
mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_mxm_vc_init(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vc_ch = &vc->ch;
MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc);
MPIDI_STATE_DECL(MPID_STATE_MXM_VC_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MXM_VC_INIT);
/* local connection is used for any source communication */
MPIU_Assert(MPID_nem_mem_region.rank != vc->lpid);
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE,
(MPIU_DBG_FDEST,
"[%i]=== connecting to %i \n", MPID_nem_mem_region.rank, vc->lpid));
{
char *business_card;
int val_max_sz;
#ifdef USE_PMI2_API
val_max_sz = PMI2_MAX_VALLEN;
#else
mpi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
#endif
business_card = (char *) MPIU_Malloc(val_max_sz);
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, val_max_sz, vc->pg);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
vc_area->ctx = vc;
vc_area->mxm_ep = &_mxm_obj.endpoint[vc->pg_rank];
mpi_errno = _mxm_connect(&_mxm_obj.endpoint[vc->pg_rank], business_card, vc_area);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
MPIU_Free(business_card);
}
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
vc_area->pending_sends = 0;
vc->rndvSend_fn = NULL;
vc->rndvRecv_fn = NULL;
vc->sendNoncontig_fn = MPID_nem_mxm_SendNoncontig;
vc->comm_ops = &comm_ops;
vc_ch->iStartContigMsg = MPID_nem_mxm_iStartContigMsg;
vc_ch->iSendContig = MPID_nem_mxm_iSendContig;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MXM_VC_INIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int _mxm_handle_sreq(MPID_Request * req)
{
int complete = FALSE;
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length >
16 ? 16 : req_area->iov_buf[0].length));
vc_area->pending_sends -= 1;
if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))) {
MPIU_Free(req->dev.tmpbuf);
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
MPIU_Free(req_area->iov_buf);
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
reqFn = req->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(req);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
MPIDI_VC_t *vc = req->ch.vc;
reqFn(vc, req, &complete);
if (!complete) {
MPIU_Assert(complete == TRUE);
}
}
return complete;
}
static void _mxm_send_completion_cb(void *context)
{
MPID_Request *req = (MPID_Request *) context;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIU_Assert(req);
_dbg_mxm_out_req(req);
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status);
list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue);
_dbg_mxm_output(5, "========> %s SEND req %p status %d\n",
(MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"),
req, req->status.MPI_ERROR);
if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) {
_mxm_handle_sreq(req);
}
}
int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype datatype,
int rank, int tag, MPID_Comm * comm, int context_offset,
MPID_Request ** sreq_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq = NULL;
MPID_Datatype *dt_ptr;
int dt_contig;
MPIDI_msg_sz_t data_sz;
MPI_Aint dt_true_lb;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISSEND);
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
/* create a request */
MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit);
MPIU_Assert(sreq != NULL);
MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPID_Datatype_get_ptr(datatype, sreq->dev.datatype_ptr);
MPID_Datatype_add_ref(sreq->dev.datatype_ptr);
}
sreq->partner_request = NULL;
sreq->dev.OnDataAvail = NULL;
sreq->dev.tmpbuf = NULL;
sreq->ch.vc = vc;
sreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"isSend ========> Sending USER msg for req %p (context %d to %d tag %d size %d) \n",
sreq, comm->context_id + context_offset, rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area-> ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (data_sz) {
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
MPIDI_msg_sz_t last;
MPI_Aint packsize = 0;
sreq->ch.noncontig = TRUE;
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER,
"**nomem", "**nomem %s", "MPID_Segment_alloc");
MPIR_Pack_size_impl(count, datatype, &packsize);
last = data_sz;
if (packsize > 0) {
sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize);
MPIU_Assert(sreq->dev.tmpbuf);
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf);
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = sreq->dev.tmpbuf;
req_area->iov_buf[0].length = last;
}
}
}
vc_area->pending_sends += 1;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC,
(mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag,
comm->context_id
+
context_offset),
0);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
_dbg_mxm_out_req(sreq);
fn_exit:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISSEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int _mxm_handle_rreq(MPID_Request * req)
{
int complete = FALSE, found = FALSE;
int dt_contig;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_msg_sz_t userbuf_sz;
MPID_Datatype *dt_ptr;
MPIDI_msg_sz_t data_sz;
MPID_nem_mxm_vc_area *vc_area ATTRIBUTE((unused)) = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
void *tmp_buf = NULL;
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_MSGQ_MUTEX);
found = MPIDI_CH3U_Recvq_DP(req);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_MSGQ_MUTEX);
/* an MPI_ANY_SOURCE request may have been previously removed from the
* CH3 queue by an FDP (find and dequeue posted) operation */
if (req->dev.match.parts.rank != MPI_ANY_SOURCE) {
MPIU_Assert(found);
}
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr,
dt_true_lb);
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length));
if (req->dev.recv_data_sz <= userbuf_sz) {
data_sz = req->dev.recv_data_sz;
if (req->status.MPI_ERROR == MPI_ERR_TRUNCATE) {
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TRUNCATE, "**truncate",
"**truncate %d %d %d %d",
req->status.MPI_SOURCE,
req->status.MPI_TAG, req->dev.recv_data_sz,
userbuf_sz);
}
}
else {
data_sz = userbuf_sz;
MPIR_STATUS_SET_COUNT(req->status, userbuf_sz);
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST,
"receive buffer too small; message truncated, msg_sz="
MPIDI_MSG_SZ_FMT ", userbuf_sz="
MPIDI_MSG_SZ_FMT, req->dev.recv_data_sz, userbuf_sz));
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TRUNCATE, "**truncate",
"**truncate %d %d %d %d",
req->status.MPI_SOURCE, req->status.MPI_TAG,
req->dev.recv_data_sz, userbuf_sz);
}
if (!dt_contig) {
MPIDI_msg_sz_t last = 0;
if (req->dev.tmpbuf != NULL) {
last = req->dev.recv_data_sz;
MPID_Segment_unpack(req->dev.segment_ptr, 0, &last, req->dev.tmpbuf);
tmp_buf = req->dev.tmpbuf;
}
else {
mxm_req_buffer_t *iov_buf;
MPL_IOV *iov;
int n_iov = 0;
int index;
last = req->dev.recv_data_sz;
n_iov = req_area->iov_count;
iov_buf = req_area->iov_buf;
if (last && n_iov > 0) {
iov = MPIU_Malloc(n_iov * sizeof(*iov));
MPIU_Assert(iov);
for (index = 0; index < n_iov; index++) {
iov[index].MPL_IOV_BUF = iov_buf[index].ptr;
iov[index].MPL_IOV_LEN = iov_buf[index].length;
}
MPID_Segment_unpack_vector(req->dev.segment_ptr, req->dev.segment_first, &last, iov,
&n_iov);
MPIU_Free(iov);
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
tmp_buf = req_area->iov_buf;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
}
if (last != data_sz) {
MPIR_STATUS_SET_COUNT(req->status, last);
if (req->dev.recv_data_sz <= userbuf_sz) {
/* If the data can't be unpacked, the we have a
* mismatch between the datatype and the amount of
* data received. Throw away received data.
*/
MPIR_ERR_SETSIMPLE(req->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
}
}
MPIDI_CH3U_Handle_recv_req(req->ch.vc, req, &complete);
MPIU_Assert(complete == TRUE);
if (tmp_buf)
MPIU_Free(tmp_buf);
return complete;
}
int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_RECV);
MPIU_Assert(rreq);
MPIU_Assert(((rreq->dev.match.parts.rank == MPI_ANY_SOURCE) && (vc == NULL)) ||
(vc && !vc->ch.is_local));
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz,
dt_ptr, dt_true_lb);
{
MPIU_Context_id_t context_id = rreq->dev.match.parts.context_id;
int tag = rreq->dev.match.parts.tag;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
mxm_mq_h *mq_h_v = (mxm_mq_h *) rreq->comm->dev.ch.netmod_priv;
rreq->dev.OnDataAvail = NULL;
rreq->dev.tmpbuf = NULL;
rreq->ch.vc = vc;
rreq->ch.noncontig = FALSE;
_dbg_mxm_output(5,
"Recv ========> Getting USER msg for req %p (context %d from %d tag %d size %d) \n",
rreq, context_id, rreq->dev.match.parts.rank, tag, data_sz);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(rreq);
req_area->ctx = rreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[0].ptr = NULL;
req_area->iov_buf[0].length = 0;
if (dt_contig) {
req_area->iov_count = 1;
req_area->iov_buf[0].ptr = (char *) (rreq->dev.user_buf) + dt_true_lb;
req_area->iov_buf[0].length = data_sz;
}
else {
rreq->ch.noncontig = TRUE;
mpi_errno = _mxm_process_rdtype(&rreq, rreq->dev.datatype, dt_ptr, data_sz,
rreq->dev.user_buf, rreq->dev.user_count,
&req_area->iov_buf, &req_area->iov_count);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
mpi_errno = _mxm_irecv((vc_area ? vc_area->mxm_ep : NULL), req_area,
tag,
(rreq->comm ? mq_h_v[0] : mxm_obj->mxm_mq), _mxm_tag_mpi2mxm(tag,
context_id));
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (vc)
_dbg_mxm_out_req(rreq);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_RECV);
return mpi_errno;
fn_fail:ATTRIBUTE((unused))
goto fn_exit;
}
int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
MPIDI_msg_sz_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t last;
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "MPID_nem_mxm_iSendNoncontig");
MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t));
_dbg_mxm_output(5,
"SendNoncontig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n",
vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t),
sreq->dev.segment_size - sreq->dev.segment_first);
vc_area = VC_BASE(vc);
req_area = REQ_BASE(sreq);
req_area->ctx = sreq;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
req_area->iov_buf[req_area->iov_count].ptr = (void *) &(sreq->dev.pending_pkt);
req_area->iov_buf[req_area->iov_count].length = sizeof(MPIDI_CH3_Pkt_t);
(req_area->iov_count)++;
if (sreq->dev.ext_hdr_ptr != NULL) {
req_area->iov_buf[req_area->iov_count].ptr = (void *) (sreq->dev.ext_hdr_ptr);
req_area->iov_buf[req_area->iov_count].length = sreq->dev.ext_hdr_sz;
(req_area->iov_count)++;
}
last = sreq->dev.segment_size;
/* NOTE: currently upper layer never pass packet with data that has
* either "last <= 0" or "last-sreq->dev.segment_first <=0" to this
* layer. In future, if upper layer passes such kind of packet, the
* judgement of the following IF branch needs to be modified. */
MPIU_Assert(last > 0 && last - sreq->dev.segment_first > 0);
if (last > 0) {
sreq->dev.tmpbuf = MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first));
MPIU_Assert(sreq->dev.tmpbuf);
MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.tmpbuf);
MPIU_Assert(last == sreq->dev.segment_size);
req_area->iov_buf[req_area->iov_count].ptr = sreq->dev.tmpbuf;
req_area->iov_buf[req_area->iov_count].length = last - sreq->dev.segment_first;
(req_area->iov_count)++;
}
vc_area->pending_sends += 1;
sreq->ch.vc = vc;
sreq->ch.noncontig = TRUE;
mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM,
mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
