static void dequeue_req(const ptl_event_t *e)
{
int found;
MPID_Request *const rreq = e->user_ptr;
MPI_Aint s_len, r_len;
/* At this point we know the ME is unlinked. Invalidate the handle to
prevent further accesses, e.g. an attempted cancel. */
REQ_PTL(rreq)->put_me = PTL_INVALID_HANDLE;
found = MPIDI_CH3U_Recvq_DP(rreq);
/* an MPI_ANY_SOURCE request may have been previously removed from the
CH3 queue by an FDP (find and dequeue posted) operation */
if (rreq->dev.match.parts.rank != MPI_ANY_SOURCE)
MPIU_Assert(found);
rreq->status.MPI_ERROR = MPI_SUCCESS;
rreq->status.MPI_SOURCE = NPTL_MATCH_GET_RANK(e->match_bits);
rreq->status.MPI_TAG = NPTL_MATCH_GET_TAG(e->match_bits);
MPID_Datatype_get_size_macro(rreq->dev.datatype, r_len);
r_len *= rreq->dev.user_count;
s_len = NPTL_HEADER_GET_LENGTH(e->hdr_data);
if (s_len > r_len) {
/* truncated data */
MPIR_STATUS_SET_COUNT(rreq->status, r_len);
MPIR_ERR_SET2(rreq->status.MPI_ERROR, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", s_len, r_len);
} else {
MPIR_STATUS_SET_COUNT(rreq->status, s_len);
}
}
int MPID_Cancel_recv(MPIR_Request * rreq)
{
int netmod_cancelled = TRUE;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_CANCEL_RECV);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_CANCEL_RECV);
MPIR_Assert(rreq->kind == MPIR_REQUEST_KIND__RECV);
/* If the netmod has its own cancel_recv function, we need to call
it here. ANYSOURCE cancels (netmod and otherwise) are handled by
MPIDI_CH3U_Recvq_DP below. */
#ifdef ENABLE_COMM_OVERRIDES
if (rreq->dev.match.parts.rank != MPI_ANY_SOURCE)
{
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc_set_active(rreq->comm, rreq->dev.match.parts.rank, &vc);
if (vc->comm_ops && vc->comm_ops->cancel_recv)
netmod_cancelled = !vc->comm_ops->cancel_recv(NULL, rreq);
}
#endif
if (netmod_cancelled && MPIDI_CH3U_Recvq_DP(rreq))
{
MPL_DBG_MSG_P(MPIDI_CH3_DBG_OTHER,VERBOSE,
"request 0x%08x cancelled", rreq->handle);
MPIR_STATUS_SET_CANCEL_BIT(rreq->status, TRUE);
MPIR_STATUS_SET_COUNT(rreq->status, 0);
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else
{
MPL_DBG_MSG_P(MPIDI_CH3_DBG_OTHER,VERBOSE,
"request 0x%08x already matched, unable to cancel", rreq->handle);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_CANCEL_RECV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Cancel_recv(MPID_Request * rreq)
{
MPIDI_STATE_DECL(MPID_STATE_MPID_CANCEL_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_CANCEL_RECV);
MPIU_Assert(rreq->kind == MPID_REQUEST_RECV);
#if defined (_OSU_PSM_)
rreq->psm_flags |= PSM_RECV_CANCEL;
if(psm_do_cancel(rreq) == MPI_SUCCESS) {
*(rreq->cc_ptr) = 0;
// MPID_Request_release(rreq);
}
goto fn_exit;
#endif
#if defined(_OSU_MVAPICH_)
/* OSU-MPI2 requires extra step to finish rndv request */
MPIDI_CH3I_MRAILI_RREQ_RNDV_FINISH(rreq);
#endif /* defined(_OSU_MVAPICH_) */
if (MPIDI_CH3U_Recvq_DP(rreq))
{
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
"request 0x%08x cancelled", rreq->handle);
rreq->status.cancelled = TRUE;
rreq->status.count = 0;
MPID_REQUEST_SET_COMPLETED(rreq);
MPID_Request_release(rreq);
}
else
{
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
"request 0x%08x already matched, unable to cancel", rreq->handle);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_CANCEL_RECV);
return MPI_SUCCESS;
}
int MPID_nem_mx_cancel_recv(MPIDI_VC_t *vc, MPID_Request *rreq)
{
mx_request_t *mx_request = NULL;
mx_return_t ret;
uint32_t result;
int mpi_errno = MPI_SUCCESS;
int handled = FALSE;
mx_request = &(REQ_FIELD(rreq,mx_request));
/* FIXME this test is probably not correct with multiple netmods */
/* We need to know to which netmod a recv request actually "belongs" to */
if(mx_request != NULL)
{
ret = mx_cancel(MPID_nem_mx_local_endpoint,mx_request,&result);
MPIU_ERR_CHKANDJUMP1(ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_cancel", "**mx_cancel %s", mx_strerror(ret));
if (result)
{
int found;
rreq->status.cancelled = TRUE;
found = MPIDI_CH3U_Recvq_DP(rreq);
MPIU_Assert(found);
rreq->status.count = 0;
MPID_REQUEST_SET_COMPLETED(rreq);
MPID_Request_release(rreq);
}
else
{
rreq->status.cancelled = FALSE;
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
"request 0x%08x already matched, unable to cancel", rreq->handle);
}
handled = TRUE;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int
MPID_nem_newmad_poll(int in_blocking_poll)
{
nm_sr_request_t *p_request = NULL;
nm_tag_t match_info = 0;
int mpi_errno = MPI_SUCCESS;
nm_sr_send_success(mpid_nem_newmad_session, &p_request);
if (p_request != NULL)
{
MPID_nem_newmad_unified_req_t *ref;
MPID_Request *req;
MPID_Request_kind_t kind;
MPIR_Context_id_t ctxt;
nm_sr_get_stag(mpid_nem_newmad_session,p_request, &match_info);
NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
nm_sr_get_ref(mpid_nem_newmad_session,p_request,(void *)&ref);
req = &(ref->mpi_req);
MPIU_Assert(req != NULL);
kind = req->kind;
if(ctxt == NEM_NMAD_INTRA_CTXT)
{
if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
{
MPID_nem_newmad_handle_sreq(req);
}
}
else
{
if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
{
MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
MPID_nem_newmad_handle_sreq(req);
}
}
}
nm_sr_recv_success(mpid_nem_newmad_session, &p_request);
if (p_request != NULL)
{
MPID_nem_newmad_unified_req_t *ref;
MPID_Request *req;
MPID_Request_kind_t kind;
MPIR_Context_id_t ctxt;
size_t size;
nm_sr_get_ref(mpid_nem_newmad_session,p_request,(void *)&ref);
req = &(ref->mpi_req);
MPIU_Assert(req != NULL);
kind = req->kind;
nm_sr_get_size(mpid_nem_newmad_session, p_request, &size);
nm_sr_get_rtag(mpid_nem_newmad_session,p_request, &match_info);
NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
if(ctxt == NEM_NMAD_INTRA_CTXT)
{
MPID_nem_newmad_internal_req_t *adi_req = &(ref->nem_newmad_req);
if (kind == MPID_REQUEST_RECV)
{
if (size <= sizeof(MPIDI_CH3_PktGeneric_t))
{
MPID_nem_handle_pkt(adi_req->vc,(char *)&(adi_req->pending_pkt),(MPIDI_msg_sz_t)(size));
}
else
{
MPID_nem_handle_pkt(adi_req->vc,(char *)(adi_req->tmpbuf),(MPIDI_msg_sz_t)(adi_req->tmpbuf_sz));
MPIU_Free(adi_req->tmpbuf);
}
nm_core_disable_progression(mpid_nem_newmad_session->p_core);
MPID_nem_newmad_internal_req_enqueue(adi_req);
nm_core_enable_progression(mpid_nem_newmad_session->p_core);
}
else
{
MPIU_Assert(0);
}
}
else
{
if ((kind == MPID_REQUEST_RECV) || (kind == MPID_PREQUEST_RECV))
{
int found = FALSE;
nm_sr_request_t *nmad_request = NULL;
MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIU_THREAD_CS_ENTER(MSGQUEUE,req);
MPID_NEM_NMAD_GET_REQ_FROM_HASH(req,nmad_request);
if(nmad_request != NULL)
{
MPIU_Assert(req->dev.match.parts.rank == MPI_ANY_SOURCE);
MPIU_Free(nmad_request);
}
found = MPIDI_CH3U_Recvq_DP(req);
if(found){
MPID_nem_newmad_handle_rreq(req,match_info,size);
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,req);
}
else
{
fprintf(stdout, ">>>>>>>>>>>>> ERROR: Wrong req type : %i (%p)\n",(int)kind,req);
MPIU_Assert(0);
}
}
}
fn_exit:
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
static int _mxm_handle_rreq(MPID_Request * req)
{
int complete = FALSE, found = FALSE;
int dt_contig;
MPI_Aint dt_true_lb ATTRIBUTE((unused));
MPIDI_msg_sz_t userbuf_sz;
MPID_Datatype *dt_ptr;
MPIDI_msg_sz_t data_sz;
MPID_nem_mxm_vc_area *vc_area ATTRIBUTE((unused)) = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
void *tmp_buf = NULL;
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_MSGQ_MUTEX);
found = MPIDI_CH3U_Recvq_DP(req);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_MSGQ_MUTEX);
/* an MPI_ANY_SOURCE request may have been previously removed from the
* CH3 queue by an FDP (find and dequeue posted) operation */
if (req->dev.match.parts.rank != MPI_ANY_SOURCE) {
MPIU_Assert(found);
}
MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, userbuf_sz, dt_ptr,
dt_true_lb);
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length > 16 ? 16 : req_area->iov_buf[0].length));
if (req->dev.recv_data_sz <= userbuf_sz) {
data_sz = req->dev.recv_data_sz;
if (req->status.MPI_ERROR == MPI_ERR_TRUNCATE) {
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TRUNCATE, "**truncate",
"**truncate %d %d %d %d",
req->status.MPI_SOURCE,
req->status.MPI_TAG, req->dev.recv_data_sz,
userbuf_sz);
}
}
else {
data_sz = userbuf_sz;
MPIR_STATUS_SET_COUNT(req->status, userbuf_sz);
MPIU_DBG_MSG_FMT(CH3_OTHER, VERBOSE, (MPIU_DBG_FDEST,
"receive buffer too small; message truncated, msg_sz="
MPIDI_MSG_SZ_FMT ", userbuf_sz="
MPIDI_MSG_SZ_FMT, req->dev.recv_data_sz, userbuf_sz));
req->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_TRUNCATE, "**truncate",
"**truncate %d %d %d %d",
req->status.MPI_SOURCE, req->status.MPI_TAG,
req->dev.recv_data_sz, userbuf_sz);
}
if (!dt_contig) {
MPIDI_msg_sz_t last = 0;
if (req->dev.tmpbuf != NULL) {
last = req->dev.recv_data_sz;
MPID_Segment_unpack(req->dev.segment_ptr, 0, &last, req->dev.tmpbuf);
tmp_buf = req->dev.tmpbuf;
}
else {
mxm_req_buffer_t *iov_buf;
MPL_IOV *iov;
int n_iov = 0;
int index;
last = req->dev.recv_data_sz;
n_iov = req_area->iov_count;
iov_buf = req_area->iov_buf;
if (last && n_iov > 0) {
iov = MPIU_Malloc(n_iov * sizeof(*iov));
MPIU_Assert(iov);
for (index = 0; index < n_iov; index++) {
iov[index].MPL_IOV_BUF = iov_buf[index].ptr;
iov[index].MPL_IOV_LEN = iov_buf[index].length;
}
MPID_Segment_unpack_vector(req->dev.segment_ptr, req->dev.segment_first, &last, iov,
&n_iov);
MPIU_Free(iov);
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
tmp_buf = req_area->iov_buf;
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
}
if (last != data_sz) {
MPIR_STATUS_SET_COUNT(req->status, last);
if (req->dev.recv_data_sz <= userbuf_sz) {
/* If the data can't be unpacked, the we have a
* mismatch between the datatype and the amount of
* data received. Throw away received data.
*/
MPIR_ERR_SETSIMPLE(req->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
}
}
MPIDI_CH3U_Handle_recv_req(req->ch.vc, req, &complete);
MPIU_Assert(complete == TRUE);
if (tmp_buf)
MPIU_Free(tmp_buf);
return complete;
}
