/* MSGQUEUE lock is not held */ void MPIDI_Callback_process_trunc(pami_context_t context, MPID_Request *rreq, pami_recv_t *recv, const void *sndbuf) { rreq->status.MPI_ERROR = MPI_ERR_TRUNCATE; /* -------------------------------------------------------------- */ /* The data is already available, so we can just unpack it now. */ /* -------------------------------------------------------------- */ if (recv) { MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE); rreq->mpid.uebuflen = MPIR_STATUS_GET_COUNT(rreq->status); rreq->mpid.uebuf = MPIU_Malloc(MPIR_STATUS_GET_COUNT(rreq->status)); MPID_assert(rreq->mpid.uebuf != NULL); rreq->mpid.uebuf_malloc = mpiuMalloc; recv->addr = rreq->mpid.uebuf; } else { MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE); rreq->mpid.uebuflen = MPIR_STATUS_GET_COUNT(rreq->status); rreq->mpid.uebuf = (void*)sndbuf; MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS); MPID_Request_release(rreq); } }
/* MSGQUEUE lock is not held */ void MPIDI_Callback_process_userdefined_dt(pami_context_t context, const void * sndbuf, size_t sndlen, MPID_Request * rreq) { unsigned dt_contig, dt_size; MPID_Datatype *dt_ptr; MPI_Aint dt_true_lb; MPIDI_Datatype_get_info(rreq->mpid.userbufcount, rreq->mpid.datatype, dt_contig, dt_size, dt_ptr, dt_true_lb); /* ----------------------------- */ /* Test for truncated message. */ /* ----------------------------- */ if (unlikely(sndlen > dt_size)) { #if ASSERT_LEVEL > 0 MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf); return; #else sndlen = dt_size; #endif } /* * This is to test that the fields don't need to be * initialized. Remove after this doesn't fail for a while. */ if (likely (dt_contig)) { MPID_assert(rreq->mpid.uebuf == NULL); MPID_assert(rreq->mpid.uebuflen == 0); void* rcvbuf = rreq->mpid.userbuf + dt_true_lb;; memcpy(rcvbuf, sndbuf, sndlen); MPIDI_Request_complete(rreq); return; } MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE); rreq->mpid.uebuflen = sndlen; rreq->mpid.uebuf = (void*)sndbuf; MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS); MPID_Request_release(rreq); }
static inline void MPIDI_RecvShortCB(pami_context_t context, const void * _msginfo, const void * sndbuf, size_t sndlen, pami_endpoint_t sender, unsigned isSync) { MPID_assert(_msginfo != NULL); const MPIDI_MsgInfo *msginfo = (const MPIDI_MsgInfo *)_msginfo; MPID_Request * rreq = NULL; pami_task_t source; #if TOKEN_FLOW_CONTROL int rettoks=0; #endif /* -------------------- */ /* Match the request. */ /* -------------------- */ unsigned rank = msginfo->MPIrank; unsigned tag = msginfo->MPItag; unsigned context_id = msginfo->MPIctxt; MPIU_THREAD_CS_ENTER(MSGQUEUE,0); source = PAMIX_Endpoint_query(sender); MPIDI_Receive_tokens(msginfo,source); #ifndef OUT_OF_ORDER_HANDLING rreq = MPIDI_Recvq_FDP(rank, tag, context_id); #else rreq = MPIDI_Recvq_FDP(rank, source, tag, context_id, msginfo->MPIseqno); #endif /* Match not found */ if (unlikely(rreq == NULL)) { #if (MPIDI_STATISTICS) MPID_NSTAT(mpid_statp->earlyArrivals); #endif MPIU_THREAD_CS_EXIT(MSGQUEUE,0); MPID_Request *newreq = MPIDI_Request_create2(); MPID_assert(newreq != NULL); if (sndlen) { newreq->mpid.uebuflen = sndlen; if (!TOKEN_FLOW_CONTROL_ON) { newreq->mpid.uebuf = MPL_malloc(sndlen); newreq->mpid.uebuf_malloc = mpiuMalloc; } else { #if TOKEN_FLOW_CONTROL MPIU_THREAD_CS_ENTER(MSGQUEUE,0); newreq->mpid.uebuf = MPIDI_mm_alloc(sndlen); newreq->mpid.uebuf_malloc = mpidiBufMM; MPIU_THREAD_CS_EXIT(MSGQUEUE,0); #else MPID_assert_always(0); #endif } MPID_assert(newreq->mpid.uebuf != NULL); } MPIU_THREAD_CS_ENTER(MSGQUEUE,0); #ifndef OUT_OF_ORDER_HANDLING rreq = MPIDI_Recvq_FDP(rank, tag, context_id); #else rreq = MPIDI_Recvq_FDP(rank, PAMIX_Endpoint_query(sender), tag, context_id, msginfo->MPIseqno); #endif if (unlikely(rreq == NULL)) { MPIDI_Callback_process_unexp(newreq, context, msginfo, sndlen, sender, sndbuf, NULL, isSync); /* request is always complete now */ if (TOKEN_FLOW_CONTROL_ON && sndlen) { #if TOKEN_FLOW_CONTROL MPIDI_Token_cntr[source].unmatched++; #else MPID_assert_always(0); #endif } MPIU_THREAD_CS_EXIT(MSGQUEUE,0); MPID_Request_release(newreq); goto fn_exit_short; } else { MPIU_THREAD_CS_EXIT(MSGQUEUE,0); MPID_Request_discard(newreq); } } else { #if (MPIDI_STATISTICS) MPID_NSTAT(mpid_statp->earlyArrivalsMatched); #endif if (TOKEN_FLOW_CONTROL_ON && sndlen) { #if TOKEN_FLOW_CONTROL MPIDI_Update_rettoks(source); MPIDI_Must_return_tokens(context,source); #else MPID_assert_always(0); #endif } MPIU_THREAD_CS_EXIT(MSGQUEUE,0); } /* the receive queue processing has been completed and we found match*/ /* ---------------------- */ /* Copy in information. */ /* ---------------------- */ rreq->status.MPI_SOURCE = rank; rreq->status.MPI_TAG = tag; MPIR_STATUS_SET_COUNT(rreq->status, sndlen); MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE); MPIDI_Request_cpyPeerRequestH(rreq, msginfo); MPIDI_Request_setSync (rreq, isSync); MPIDI_Request_setRzv (rreq, 0); /* ----------------------------- */ /* Request was already posted. */ /* ----------------------------- */ if (unlikely(isSync)) MPIDI_SyncAck_post(context, rreq, PAMIX_Endpoint_query(sender)); if (unlikely(HANDLE_GET_KIND(rreq->mpid.datatype) != HANDLE_KIND_BUILTIN)) { MPIDI_Callback_process_userdefined_dt(context, sndbuf, sndlen, rreq); goto fn_exit_short; } size_t dt_size = rreq->mpid.userbufcount * MPID_Datatype_get_basic_size(rreq->mpid.datatype); /* ----------------------------- */ /* Test for truncated message. */ /* ----------------------------- */ if (unlikely(sndlen > dt_size)) { #if ASSERT_LEVEL > 0 MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf); goto fn_exit_short; #else sndlen = dt_size; #endif } MPID_assert(rreq->mpid.uebuf == NULL); MPID_assert(rreq->mpid.uebuflen == 0); void* rcvbuf = rreq->mpid.userbuf; if (sndlen > 0) { #if CUDA_AWARE_SUPPORT if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf)) { cudaError_t cudaerr = CudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice); } else #endif memcpy(rcvbuf, sndbuf, sndlen); } TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen); TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH); TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); MPIDI_Request_complete(rreq); fn_exit_short: #ifdef OUT_OF_ORDER_HANDLING MPIU_THREAD_CS_ENTER(MSGQUEUE,0); if (MPIDI_In_cntr[source].n_OutOfOrderMsgs>0) { MPIDI_Recvq_process_out_of_order_msgs(source, context); } MPIU_THREAD_CS_EXIT(MSGQUEUE,0); #endif /* ---------------------------------------- */ /* Signal that the recv has been started. */ /* ---------------------------------------- */ MPIDI_Progress_signal(); }
/* MSGQUEUE lock must be held by caller */ void MPIDI_Callback_process_unexp(MPID_Request *newreq, pami_context_t context, const MPIDI_MsgInfo * msginfo, size_t sndlen, pami_endpoint_t sender, const void * sndbuf, pami_recv_t * recv, unsigned isSync) { MPID_Request *rreq = NULL; /* ---------------------------------------------------- */ /* Fallback position: */ /* + Request was not posted, or */ /* + Request was long & not contiguous. */ /* We must allocate enough space to hold the message. */ /* The temporary buffer will be unpacked later. */ /* ---------------------------------------------------- */ unsigned rank = msginfo->MPIrank; unsigned tag = msginfo->MPItag; unsigned context_id = msginfo->MPIctxt; #ifndef OUT_OF_ORDER_HANDLING rreq = MPIDI_Recvq_AEU(newreq, rank, tag, context_id); #else unsigned msg_seqno = msginfo->MPIseqno; rreq = MPIDI_Recvq_AEU(newreq, rank, PAMIX_Endpoint_query(sender), tag, context_id, msg_seqno); #endif /* ---------------------- */ /* Copy in information. */ /* ---------------------- */ rreq->status.MPI_SOURCE = rank; rreq->status.MPI_TAG = tag; MPIR_STATUS_SET_COUNT(rreq->status, sndlen); MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE); MPIDI_Request_cpyPeerRequestH(rreq, msginfo); MPIDI_Request_setSync (rreq, isSync); /* Set the rank of the sender if a sync msg. */ #ifndef OUT_OF_ORDER_HANDLING if (isSync) { #endif MPIDI_Request_setPeerRank_comm(rreq, rank); MPIDI_Request_setPeerRank_pami(rreq, PAMIX_Endpoint_query(sender)); #ifndef OUT_OF_ORDER_HANDLING } #endif MPID_assert(!sndlen || rreq->mpid.uebuf != NULL); TRACE_MEMSET_R(PAMIX_Endpoint_query(sender),msg_seqno,recv_status); TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),msgid,msginfo->MPIseqno); TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rtag,tag); TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rctx,msginfo->MPIctxt); TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rlen,sndlen); TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),fl.f.sync,isSync); TRACE_SET_R_VAL(PAMIX_Endpoint_query(sender),(msginfo->MPIseqno & SEQMASK),rsource,PAMIX_Endpoint_query(sender)); TRACE_SET_REQ_VAL(rreq->mpid.idx,(msginfo->MPIseqno & SEQMASK)); if (recv != NULL) { recv->local_fn = MPIDI_RecvDoneCB_mutexed; recv->cookie = rreq; /* -------------------------------------------------- */ /* Let PAMI know where to put the rest of the data. */ /* -------------------------------------------------- */ recv->addr = rreq->mpid.uebuf; } else { /* ------------------------------------------------- */ /* We have the data; copy it and complete the msg. */ /* ------------------------------------------------- */ memcpy(rreq->mpid.uebuf, sndbuf, sndlen); MPIDI_RecvDoneCB(context, rreq, PAMI_SUCCESS); /* caller must release rreq, after unlocking MSGQUEUE */ } }
void MPIDI_RecvMsg_Unexp(MPID_Request * rreq, void * buf, int count, MPI_Datatype datatype) { /* ------------------------------------------------------------ */ /* message was found in unexpected queue */ /* ------------------------------------------------------------ */ /* We must acknowledge synchronous send requests */ /* The recvnew callback will acknowledge the posted messages */ /* Recv functions will ack the messages that are unexpected */ /* ------------------------------------------------------------ */ #ifdef MPIDI_TRACE MPIDI_In_cntr[(rreq->mpid.partner_id)].R[(rreq->mpid.idx)].matchedInUQ=1; #endif if (MPIDI_Request_isRzv(rreq)) { const unsigned is_sync = MPIDI_Request_isSync(rreq); const unsigned is_zero = (rreq->mpid.envelope.length==0); /* -------------------------------------------------------- */ /* Received an expected flow-control rendezvous RTS. */ /* This is very similar to the found/incomplete case */ /* -------------------------------------------------------- */ if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { MPID_Datatype_get_ptr(datatype, rreq->mpid.datatype_ptr); MPID_Datatype_add_ref(rreq->mpid.datatype_ptr); } if (likely((is_sync+is_zero) == 0)) MPIDI_Context_post(MPIDI_Context_local(rreq), &rreq->mpid.post_request, MPIDI_RendezvousTransfer, rreq); else if (is_sync != 0) MPIDI_Context_post(MPIDI_Context_local(rreq), &rreq->mpid.post_request, MPIDI_RendezvousTransfer_SyncAck, rreq); else MPIDI_Context_post(MPIDI_Context_local(rreq), &rreq->mpid.post_request, MPIDI_RendezvousTransfer_zerobyte, rreq); } else { if (MPID_cc_is_complete(&rreq->cc)) { if (unlikely(MPIDI_Request_isSync(rreq))) { /* Post this to the context for asynchronous progresss. We cannot do * the send-immediate inline here because we may not have the * context locked (its is being asynchrously advanced). * Must "uncomplete" the message (increment the ref and completion counts) so we * hold onto this request object until this send has completed. When MPIDI_SyncAck_handoff * finishes sending the ack, it will complete the request, decrementing the ref and * completion counts. */ MPIDI_Request_uncomplete(rreq); MPIDI_Send_post(MPIDI_SyncAck_handoff, rreq); } /* -------------------------------- */ /* request is complete */ /* -------------------------------- */ if (rreq->mpid.uebuf != NULL) { if (likely(rreq->status.cancelled == FALSE)) { MPIDI_msg_sz_t _count=0; MPIDI_Buffer_copy(rreq->mpid.uebuf, rreq->mpid.uebuflen, MPI_CHAR, &rreq->status.MPI_ERROR, buf, count, datatype, &_count, &rreq->status.MPI_ERROR); rreq->status.count = _count; } } else { MPID_assert(rreq->mpid.uebuflen == 0); rreq->status.count = 0; } } else { /* -------------------------------- */ /* request is incomplete */ /* -------------------------------- */ if (unlikely(MPIDI_Request_isSync(rreq))) { /* Post this to the context for asynchronous progresss. We cannot do * the send-immediate inline here because we may not have the * context locked (its is being asynchrously advanced). * Must "uncomplete" the message (increment the ref and completion counts) so we * hold onto this request object until this send has completed. When MPIDI_SyncAck_handoff * finishes sending the ack, it will complete the request, decrementing the ref and * completion counts. */ MPIDI_Request_uncomplete(rreq); MPIDI_Send_post(MPIDI_SyncAck_handoff, rreq); } if(rreq->status.cancelled == FALSE) { MPIDI_Request_setCA(rreq, MPIDI_CA_UNPACK_UEBUF_AND_COMPLETE); } if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) { MPID_Datatype_get_ptr(datatype, rreq->mpid.datatype_ptr); MPID_Datatype_add_ref(rreq->mpid.datatype_ptr); } } } }