int MPID_nem_mxm_vc_terminate(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MXM_VC_TERMINATE); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MXM_VC_TERMINATE); if (vc->state != MPIDI_VC_STATE_CLOSED) { /* VC is terminated as a result of a fault. Complete * outstanding sends with an error and terminate connection * immediately. */ MPIR_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank); } else { while (vc_area->pending_sends > 0) MPID_nem_mxm_poll(FALSE); } mpi_errno = MPIDI_CH3U_Handle_connection(vc, MPIDI_VC_EVENT_TERMINATED); if (mpi_errno) MPIR_ERR_POP(mpi_errno); fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MXM_VC_TERMINATE); return mpi_errno; fn_fail: goto fn_exit; }
int _mxm_handle_sreq(MPID_Request * req) { int complete = FALSE; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; vc_area = VC_BASE(req->ch.vc); req_area = REQ_BASE(req); _dbg_mxm_out_buf(req_area->iov_buf[0].ptr, (req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length)); vc_area->pending_sends -= 1; if (req->dev.tmpbuf) { if (req->dev.datatype_ptr || req->ch.noncontig) { MPIU_Free(req->dev.tmpbuf); } } if (req_area->iov_count > MXM_MPICH_MAX_IOV) { MPIU_Free(req_area->iov_buf); req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; } MPIDI_CH3U_Handle_send_req(req->ch.vc, req, &complete); MPIU_Assert(complete == TRUE); return complete; }
int MPID_nem_mxm_vc_destroy(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MXM_VC_DESTROY); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MXM_VC_DESTROY); /* Do nothing because * finalize is called before vc destroy as result it is not possible * to destroy endpoint here */ #if 0 MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); if (vc_area->ctx == vc) { mpi_errno = _mxm_disconnect(vc_area->mxm_ep); if (mpi_errno) MPIR_ERR_POP(mpi_errno); } #endif fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MXM_VC_DESTROY); return mpi_errno; fn_fail: goto fn_exit; }
static void _mxm_recv_completion_cb(void *context) { MPID_Request *req = (MPID_Request *) context; mxm_recv_req_t *mxm_rreq; MPID_nem_mxm_req_area *req_area = NULL; MPIU_Assert(req); _dbg_mxm_out_req(req); req_area = REQ_BASE(req); _mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status); mxm_rreq = &req_area->mxm_req->item.recv; req->status.MPI_TAG = _mxm_tag_mxm2mpi(mxm_rreq->completion.sender_tag); req->status.MPI_SOURCE = mxm_rreq->completion.sender_imm; req->dev.recv_data_sz = mxm_rreq->completion.actual_len; MPIR_STATUS_SET_COUNT(req->status, req->dev.recv_data_sz); if (req->ch.vc) { MPID_nem_mxm_vc_area *vc_area = VC_BASE(req->ch.vc); list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue); } else { list_enqueue(&mxm_obj->free_queue, &req_area->mxm_req->queue); } _dbg_mxm_output(5, "========> %s RECV req %p status %d\n", (MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"), req, req->status.MPI_ERROR); if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) { _mxm_handle_rreq(req); } }
int MPID_nem_mxm_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr) { int mpi_errno = MPI_SUCCESS; MPID_Request *sreq = NULL; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mxm_iStartContigMsg"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); /* create a request */ sreq = MPID_Request_create(); MPIU_Assert(sreq != NULL); MPIU_Object_set_ref(sreq, 2); MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t)); sreq->kind = MPID_REQUEST_SEND; sreq->dev.OnDataAvail = NULL; sreq->dev.tmpbuf = NULL; _dbg_mxm_output(5, "iStartContigMsg ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n", vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), data_sz); vc_area = VC_BASE(vc); req_area = REQ_BASE(sreq); req_area->ctx = sreq; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 1; req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt); req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t); if (data_sz) { req_area->iov_count = 2; req_area->iov_buf[1].ptr = (void *) data; req_area->iov_buf[1].length = data_sz; } vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = FALSE; mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIU_ERR_POP(mpi_errno); fn_exit: *sreq_ptr = sreq; MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISTARTCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_mxm_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "mxm_iSendContig"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t)); _dbg_mxm_output(5, "iSendContig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n", vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), data_sz); vc_area = VC_BASE(vc); req_area = REQ_BASE(sreq); req_area->ctx = sreq; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; req_area->iov_buf[req_area->iov_count].ptr = (void *) &(sreq->dev.pending_pkt); req_area->iov_buf[req_area->iov_count].length = sizeof(MPIDI_CH3_Pkt_t); (req_area->iov_count)++; if (sreq->dev.ext_hdr_sz != 0) { req_area->iov_buf[req_area->iov_count].ptr = (void *) (sreq->dev.ext_hdr_ptr); req_area->iov_buf[req_area->iov_count].length = sreq->dev.ext_hdr_sz; (req_area->iov_count)++; } if (data_sz) { req_area->iov_buf[req_area->iov_count].ptr = (void *) data; req_area->iov_buf[req_area->iov_count].length = data_sz; (req_area->iov_count)++; } vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = FALSE; mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIR_ERR_POP(mpi_errno); fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISENDCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t last; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "MPID_nem_mxm_iSendNoncontig"); MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t)); _dbg_mxm_output(5, "SendNoncontig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n", vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), sreq->dev.segment_size); vc_area = VC_BASE(vc); req_area = REQ_BASE(sreq); req_area->ctx = sreq; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 1; req_area->iov_buf[0].ptr = (void *) &(sreq->dev.pending_pkt); req_area->iov_buf[0].length = sizeof(MPIDI_CH3_Pkt_t); MPIU_Assert(sreq->dev.segment_first == 0); last = sreq->dev.segment_size; if (last > 0) { sreq->dev.tmpbuf = MPIU_Malloc((size_t) sreq->dev.segment_size); MPIU_Assert(sreq->dev.tmpbuf); MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.tmpbuf); MPIU_Assert(last == sreq->dev.segment_size); req_area->iov_count = 2; req_area->iov_buf[1].ptr = sreq->dev.tmpbuf; req_area->iov_buf[1].length = last; } vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = TRUE; mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIU_ERR_POP(mpi_errno); fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_mxm_vc_init(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; MPIDI_CH3I_VC *vc_ch = &vc->ch; MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc); MPIDI_STATE_DECL(MPID_STATE_MXM_VC_INIT); MPIDI_FUNC_ENTER(MPID_STATE_MXM_VC_INIT); /* local connection is used for any source communication */ MPIU_Assert(MPID_nem_mem_region.rank != vc->lpid); MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "[%i]=== connecting to %i \n", MPID_nem_mem_region.rank, vc->lpid)); { char *business_card; int val_max_sz; #ifdef USE_PMI2_API val_max_sz = PMI2_MAX_VALLEN; #else mpi_errno = PMI_KVS_Get_value_length_max(&val_max_sz); if (mpi_errno) MPIU_ERR_POP(mpi_errno); #endif business_card = (char *) MPIU_Malloc(val_max_sz); mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, val_max_sz, vc->pg); if (mpi_errno) MPIU_ERR_POP(mpi_errno); vc_area->ctx = vc; vc_area->mxm_ep = &_mxm_obj.endpoint[vc->pg_rank]; mpi_errno = _mxm_connect(&_mxm_obj.endpoint[vc->pg_rank], business_card, vc_area); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_Free(business_card); } MPIDI_CHANGE_VC_STATE(vc, ACTIVE); vc_area->pending_sends = 0; vc->rndvSend_fn = NULL; vc->rndvRecv_fn = NULL; vc->sendNoncontig_fn = MPID_nem_mxm_SendNoncontig; vc->comm_ops = &comm_ops; vc_ch->iStartContigMsg = MPID_nem_mxm_iStartContigMsg; vc_ch->iSendContig = MPID_nem_mxm_iSendContig; fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MXM_VC_INIT); return mpi_errno; fn_fail: goto fn_exit; }
static int _mxm_handle_sreq(MPID_Request * req) { int complete = FALSE; int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; vc_area = VC_BASE(req->ch.vc); req_area = REQ_BASE(req); _dbg_mxm_out_buf(req_area->iov_buf[0].ptr, (req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length)); vc_area->pending_sends -= 1; if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))) { MPIU_Free(req->dev.tmpbuf); } if (req_area->iov_count > MXM_MPICH_MAX_IOV) { MPIU_Free(req_area->iov_buf); req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; } reqFn = req->dev.OnDataAvail; if (!reqFn) { MPIDI_CH3U_Request_complete(req); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); } else { MPIDI_VC_t *vc = req->ch.vc; reqFn(vc, req, &complete); if (!complete) { MPIU_Assert(complete == TRUE); } } return complete; }
static void _mxm_send_completion_cb(void *context) { MPID_Request *req = (MPID_Request *) context; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; MPIU_Assert(req); _dbg_mxm_out_req(req); vc_area = VC_BASE(req->ch.vc); req_area = REQ_BASE(req); _mxm_to_mpi_status(req_area->mxm_req->item.base.error, &req->status); list_enqueue(&vc_area->mxm_ep->free_queue, &req_area->mxm_req->queue); _dbg_mxm_output(5, "========> %s SEND req %p status %d\n", (MPIR_STATUS_GET_CANCEL_BIT(req->status) ? "Canceling" : "Completing"), req, req->status.MPI_ERROR); if (likely(!MPIR_STATUS_GET_CANCEL_BIT(req->status))) { _mxm_handle_sreq(req); } }
int MPID_nem_mxm_issend(MPIDI_VC_t * vc, const void *buf, int count, MPI_Datatype datatype, int rank, int tag, MPID_Comm * comm, int context_offset, MPID_Request ** sreq_ptr) { int mpi_errno = MPI_SUCCESS; MPID_Request *sreq = NULL; MPID_Datatype *dt_ptr; int dt_contig; MPIDI_msg_sz_t data_sz; MPI_Aint dt_true_lb; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_ISSEND); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_ISSEND); MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb); /* create a request */ MPIDI_Request_create_sreq(sreq, mpi_errno, goto fn_exit); MPIU_Assert(sreq != NULL); MPIDI_Request_set_type(sreq, MPIDI_REQUEST_TYPE_SEND); MPIDI_VC_FAI_send_seqnum(vc, seqnum); MPIDI_Request_set_seqnum(sreq, seqnum); if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { MPID_Datatype_get_ptr(datatype, sreq->dev.datatype_ptr); MPID_Datatype_add_ref(sreq->dev.datatype_ptr); } sreq->partner_request = NULL; sreq->dev.OnDataAvail = NULL; sreq->dev.tmpbuf = NULL; sreq->ch.vc = vc; sreq->ch.noncontig = FALSE; _dbg_mxm_output(5, "isSend ========> Sending USER msg for req %p (context %d to %d tag %d size %d) \n", sreq, comm->context_id + context_offset, rank, tag, data_sz); vc_area = VC_BASE(vc); req_area = REQ_BASE(sreq); req_area-> ctx = sreq; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; req_area->iov_buf[0].ptr = NULL; req_area->iov_buf[0].length = 0; if (data_sz) { if (dt_contig) { req_area->iov_count = 1; req_area->iov_buf[0].ptr = (char *) (buf) + dt_true_lb; req_area->iov_buf[0].length = data_sz; } else { MPIDI_msg_sz_t last; MPI_Aint packsize = 0; sreq->ch.noncontig = TRUE; sreq->dev.segment_ptr = MPID_Segment_alloc(); MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc"); MPIR_Pack_size_impl(count, datatype, &packsize); last = data_sz; if (packsize > 0) { sreq->dev.tmpbuf = MPIU_Malloc((size_t) packsize); MPIU_Assert(sreq->dev.tmpbuf); MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0); MPID_Segment_pack(sreq->dev.segment_ptr, 0, &last, sreq->dev.tmpbuf); req_area->iov_count = 1; req_area->iov_buf[0].ptr = sreq->dev.tmpbuf; req_area->iov_buf[0].length = last; } } } vc_area->pending_sends += 1; mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_SYNC, (mxm_mq_h) comm->dev.ch.netmod_priv, comm->rank, tag, _mxm_tag_mpi2mxm(tag, comm->context_id + context_offset), 0); if (mpi_errno) MPIU_ERR_POP(mpi_errno); _dbg_mxm_out_req(sreq); fn_exit: *sreq_ptr = sreq; MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_ISSEND); return mpi_errno; fn_fail: goto fn_exit; }
static int _mxm_handle_rreq(MPID_Request * req) { int complete = FALSE, found = FALSE; int dt_contig; MPI_Aint dt_true_lb ATTRIBUTE((unused)); MPIDI_msg_sz_t userbuf_sz; MPID_Datatype *dt_ptr; MPIDI_msg_sz_t data_sz; MPID_nem_mxm_vc_area *vc_area ATTRIBUTE((unused)) = NULL; MPID_nem_mxm_req_area *req_area = NULL; void *tmp_buf = NULL; MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_MSGQ_MUTEX); found = MPIDI_CH3U_Recvq_DP(req); MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_MSGQ_MUTEX); /* an MPI_ANY_SOURCE request may have been previously removed from the * CH3 queue by an FDP (find and dequeue posted) operation */ if (req->dev.match.parts.rank != MPI_ANY_SOURCE) { MPIU_Assert(found); } MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr, dt_true_lb); vc_area = VC_BASE(req->ch.vc); req_area = REQ_BASE(req); _dbg_mxm_out_buf(req_area->iov_buf[0].ptr, (req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length)); if (req->dev.recv_data_sz <= userbuf_sz) { data_sz = req->dev.recv_data_sz; if (req->status.MPI_ERROR == MPI_ERR_TRUNCATE) { req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d %d %d", req->status.MPI_SOURCE, req->status.MPI_TAG, req->dev.recv_data_sz, userbuf_sz); } } else { data_sz = userbuf_sz; MPIR_STATUS_SET_COUNT(req->status, userbuf_sz); MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST, "receive buffer too small; message truncated, msg_sz=" MPIDI_MSG_SZ_FMT ", userbuf_sz=" MPIDI_MSG_SZ_FMT, req->dev.recv_data_sz, userbuf_sz)); req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d %d %d", req->status.MPI_SOURCE, req->status.MPI_TAG, req->dev.recv_data_sz, userbuf_sz); } if (!dt_contig) { MPIDI_msg_sz_t last = 0; if (req->dev.tmpbuf != NULL) { last = req->dev.recv_data_sz; MPID_Segment_unpack(req->dev.segment_ptr, 0, &last, req->dev.tmpbuf); tmp_buf = req->dev.tmpbuf; } else { mxm_req_buffer_t *iov_buf; MPL_IOV *iov; int n_iov = 0; int index; last = req->dev.recv_data_sz; n_iov = req_area->iov_count; iov_buf = req_area->iov_buf; if (last && n_iov > 0) { iov = MPIU_Malloc(n_iov * sizeof(*iov)); MPIU_Assert(iov); for (index = 0; index < n_iov; index++) { iov[index].MPL_IOV_BUF = iov_buf[index].ptr; iov[index].MPL_IOV_LEN = iov_buf[index].length; } MPID_Segment_unpack_vector(req->dev.segment_ptr, req->dev.segment_first, &last, iov, &n_iov); MPIU_Free(iov); } if (req_area->iov_count > MXM_MPICH_MAX_IOV) { tmp_buf = req_area->iov_buf; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; } } if (last != data_sz) { MPIR_STATUS_SET_COUNT(req->status, last); if (req->dev.recv_data_sz <= userbuf_sz) { /* If the data can't be unpacked, the we have a * mismatch between the datatype and the amount of * data received. Throw away received data. */ MPIR_ERR_SETSIMPLE(req->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch"); } } } MPIDI_CH3U_Handle_recv_req(req->ch.vc, req, &complete); MPIU_Assert(complete == TRUE); if (tmp_buf) MPIU_Free(tmp_buf); return complete; }
int MPID_nem_mxm_recv(MPIDI_VC_t * vc, MPID_Request * rreq) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t data_sz; int dt_contig; MPI_Aint dt_true_lb; MPID_Datatype *dt_ptr; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_RECV); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_RECV); MPIU_Assert(rreq); MPIU_Assert(((rreq->dev.match.parts.rank == MPI_ANY_SOURCE) && (vc == NULL)) || (vc && !vc->ch.is_local)); MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb); { MPIU_Context_id_t context_id = rreq->dev.match.parts.context_id; int tag = rreq->dev.match.parts.tag; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; mxm_mq_h *mq_h_v = (mxm_mq_h *) rreq->comm->dev.ch.netmod_priv; rreq->dev.OnDataAvail = NULL; rreq->dev.tmpbuf = NULL; rreq->ch.vc = vc; rreq->ch.noncontig = FALSE; _dbg_mxm_output(5, "Recv ========> Getting USER msg for req %p (context %d from %d tag %d size %d) \n", rreq, context_id, rreq->dev.match.parts.rank, tag, data_sz); vc_area = VC_BASE(vc); req_area = REQ_BASE(rreq); req_area->ctx = rreq; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; req_area->iov_buf[0].ptr = NULL; req_area->iov_buf[0].length = 0; if (dt_contig) { req_area->iov_count = 1; req_area->iov_buf[0].ptr = (char *) (rreq->dev.user_buf) + dt_true_lb; req_area->iov_buf[0].length = data_sz; } else { rreq->ch.noncontig = TRUE; mpi_errno = _mxm_process_rdtype(&rreq, rreq->dev.datatype, dt_ptr, data_sz, rreq->dev.user_buf, rreq->dev.user_count, &req_area->iov_buf, &req_area->iov_count); if (mpi_errno) MPIR_ERR_POP(mpi_errno); } mpi_errno = _mxm_irecv((vc_area ? vc_area->mxm_ep : NULL), req_area, tag, (rreq->comm ? mq_h_v[0] : mxm_obj->mxm_mq), _mxm_tag_mpi2mxm(tag, context_id)); if (mpi_errno) MPIR_ERR_POP(mpi_errno); } if (vc) _dbg_mxm_out_req(rreq); fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_RECV); return mpi_errno; fn_fail:ATTRIBUTE((unused)) goto fn_exit; }
int MPID_nem_mxm_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t last; MPID_nem_mxm_vc_area *vc_area = NULL; MPID_nem_mxm_req_area *req_area = NULL; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "MPID_nem_mxm_iSendNoncontig"); MPIU_Memcpy(&(sreq->dev.pending_pkt), (char *) hdr, sizeof(MPIDI_CH3_Pkt_t)); _dbg_mxm_output(5, "SendNoncontig ========> Sending ADI msg (to=%d type=%d) for req %p (data_size %d, %d) \n", vc->pg_rank, sreq->dev.pending_pkt.type, sreq, sizeof(MPIDI_CH3_Pkt_t), sreq->dev.segment_size - sreq->dev.segment_first); vc_area = VC_BASE(vc); req_area = REQ_BASE(sreq); req_area->ctx = sreq; req_area->iov_buf = req_area->tmp_buf; req_area->iov_count = 0; req_area->iov_buf[req_area->iov_count].ptr = (void *) &(sreq->dev.pending_pkt); req_area->iov_buf[req_area->iov_count].length = sizeof(MPIDI_CH3_Pkt_t); (req_area->iov_count)++; if (sreq->dev.ext_hdr_ptr != NULL) { req_area->iov_buf[req_area->iov_count].ptr = (void *) (sreq->dev.ext_hdr_ptr); req_area->iov_buf[req_area->iov_count].length = sreq->dev.ext_hdr_sz; (req_area->iov_count)++; } last = sreq->dev.segment_size; /* NOTE: currently upper layer never pass packet with data that has * either "last <= 0" or "last-sreq->dev.segment_first <=0" to this * layer. In future, if upper layer passes such kind of packet, the * judgement of the following IF branch needs to be modified. */ MPIU_Assert(last > 0 && last - sreq->dev.segment_first > 0); if (last > 0) { sreq->dev.tmpbuf = MPIU_Malloc((size_t) (sreq->dev.segment_size - sreq->dev.segment_first)); MPIU_Assert(sreq->dev.tmpbuf); MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.tmpbuf); MPIU_Assert(last == sreq->dev.segment_size); req_area->iov_buf[req_area->iov_count].ptr = sreq->dev.tmpbuf; req_area->iov_buf[req_area->iov_count].length = last - sreq->dev.segment_first; (req_area->iov_count)++; } vc_area->pending_sends += 1; sreq->ch.vc = vc; sreq->ch.noncontig = TRUE; mpi_errno = _mxm_isend(vc_area->mxm_ep, req_area, MXM_MPICH_ISEND_AM, mxm_obj->mxm_mq, mxm_obj->mxm_rank, MXM_MPICH_HID_ADI_MSG, 0, 0); if (mpi_errno) MPIR_ERR_POP(mpi_errno); fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_MXM_SENDNONCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }