static void cb_barrier(void *ctxt, void *clientdata, pami_result_t err) { int *active = (int *) clientdata; TRACE_ERR("callback. enter: %d\n", (*active)); MPIDI_Progress_signal(); (*active)--; }
void MPIDI_Win_DoneCB(pami_context_t context, void * cookie, pami_result_t result) { MPIDI_Win_request *req = (MPIDI_Win_request*)cookie; ++req->win->mpid.sync.complete; if ((req->buffer_free) && (req->type == MPIDI_WIN_REQUEST_GET)) { ++req->origin.completed; if (req->origin.completed == req->target.dt.num_contig) { int mpi_errno; mpi_errno = MPIR_Localcopy(req->buffer, req->origin.dt.size, MPI_CHAR, req->origin.addr, req->origin.count, req->origin.datatype); MPID_assert(mpi_errno == MPI_SUCCESS); MPID_Datatype_release(req->origin.dt.pointer); MPIU_Free(req->buffer); req->buffer_free = 0; } } if (req->win->mpid.sync.total == req->win->mpid.sync.complete) { if (req->buffer_free) MPIU_Free(req->buffer); if (req->accum_headers) MPIU_Free(req->accum_headers); MPIU_Free(req); } MPIDI_Progress_signal(); }
static inline void MPIDI_RecvShortCB(pami_context_t context, const void * _msginfo, const void * sndbuf, size_t sndlen, pami_endpoint_t sender, unsigned isSync) { MPID_assert(_msginfo != NULL); const MPIDI_MsgInfo *msginfo = (const MPIDI_MsgInfo *)_msginfo; MPID_Request * rreq = NULL; pami_task_t source; #if TOKEN_FLOW_CONTROL int rettoks=0; #endif /* -------------------- */ /* Match the request. */ /* -------------------- */ unsigned rank = msginfo->MPIrank; unsigned tag = msginfo->MPItag; unsigned context_id = msginfo->MPIctxt; MPIU_THREAD_CS_ENTER(MSGQUEUE,0); source = PAMIX_Endpoint_query(sender); MPIDI_Receive_tokens(msginfo,source); #ifndef OUT_OF_ORDER_HANDLING rreq = MPIDI_Recvq_FDP(rank, tag, context_id); #else rreq = MPIDI_Recvq_FDP(rank, source, tag, context_id, msginfo->MPIseqno); #endif /* Match not found */ if (unlikely(rreq == NULL)) { #if (MPIDI_STATISTICS) MPID_NSTAT(mpid_statp->earlyArrivals); #endif MPIU_THREAD_CS_EXIT(MSGQUEUE,0); MPID_Request *newreq = MPIDI_Request_create2(); MPID_assert(newreq != NULL); if (sndlen) { newreq->mpid.uebuflen = sndlen; if (!TOKEN_FLOW_CONTROL_ON) { newreq->mpid.uebuf = MPL_malloc(sndlen); newreq->mpid.uebuf_malloc = mpiuMalloc; } else { #if TOKEN_FLOW_CONTROL MPIU_THREAD_CS_ENTER(MSGQUEUE,0); newreq->mpid.uebuf = MPIDI_mm_alloc(sndlen); newreq->mpid.uebuf_malloc = mpidiBufMM; MPIU_THREAD_CS_EXIT(MSGQUEUE,0); #else MPID_assert_always(0); #endif } MPID_assert(newreq->mpid.uebuf != NULL); } MPIU_THREAD_CS_ENTER(MSGQUEUE,0); #ifndef OUT_OF_ORDER_HANDLING rreq = MPIDI_Recvq_FDP(rank, tag, context_id); #else rreq = MPIDI_Recvq_FDP(rank, PAMIX_Endpoint_query(sender), tag, context_id, msginfo->MPIseqno); #endif if (unlikely(rreq == NULL)) { MPIDI_Callback_process_unexp(newreq, context, msginfo, sndlen, sender, sndbuf, NULL, isSync); /* request is always complete now */ if (TOKEN_FLOW_CONTROL_ON && sndlen) { #if TOKEN_FLOW_CONTROL MPIDI_Token_cntr[source].unmatched++; #else MPID_assert_always(0); #endif } MPIU_THREAD_CS_EXIT(MSGQUEUE,0); MPID_Request_release(newreq); goto fn_exit_short; } else { MPIU_THREAD_CS_EXIT(MSGQUEUE,0); MPID_Request_discard(newreq); } } else { #if (MPIDI_STATISTICS) MPID_NSTAT(mpid_statp->earlyArrivalsMatched); #endif if (TOKEN_FLOW_CONTROL_ON && sndlen) { #if TOKEN_FLOW_CONTROL MPIDI_Update_rettoks(source); MPIDI_Must_return_tokens(context,source); #else MPID_assert_always(0); #endif } MPIU_THREAD_CS_EXIT(MSGQUEUE,0); } /* the receive queue processing has been completed and we found match*/ /* ---------------------- */ /* Copy in information. */ /* ---------------------- */ rreq->status.MPI_SOURCE = rank; rreq->status.MPI_TAG = tag; MPIR_STATUS_SET_COUNT(rreq->status, sndlen); MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE); MPIDI_Request_cpyPeerRequestH(rreq, msginfo); MPIDI_Request_setSync (rreq, isSync); MPIDI_Request_setRzv (rreq, 0); /* ----------------------------- */ /* Request was already posted. */ /* ----------------------------- */ if (unlikely(isSync)) MPIDI_SyncAck_post(context, rreq, PAMIX_Endpoint_query(sender)); if (unlikely(HANDLE_GET_KIND(rreq->mpid.datatype) != HANDLE_KIND_BUILTIN)) { MPIDI_Callback_process_userdefined_dt(context, sndbuf, sndlen, rreq); goto fn_exit_short; } size_t dt_size = rreq->mpid.userbufcount * MPID_Datatype_get_basic_size(rreq->mpid.datatype); /* ----------------------------- */ /* Test for truncated message. */ /* ----------------------------- */ if (unlikely(sndlen > dt_size)) { #if ASSERT_LEVEL > 0 MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf); goto fn_exit_short; #else sndlen = dt_size; #endif } MPID_assert(rreq->mpid.uebuf == NULL); MPID_assert(rreq->mpid.uebuflen == 0); void* rcvbuf = rreq->mpid.userbuf; if (sndlen > 0) { #if CUDA_AWARE_SUPPORT if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf)) { cudaError_t cudaerr = CudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice); } else #endif memcpy(rcvbuf, sndbuf, sndlen); } TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen); TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH); TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); MPIDI_Request_complete(rreq); fn_exit_short: #ifdef OUT_OF_ORDER_HANDLING MPIU_THREAD_CS_ENTER(MSGQUEUE,0); if (MPIDI_In_cntr[source].n_OutOfOrderMsgs>0) { MPIDI_Recvq_process_out_of_order_msgs(source, context); } MPIU_THREAD_CS_EXIT(MSGQUEUE,0); #endif /* ---------------------------------------- */ /* Signal that the recv has been started. */ /* ---------------------------------------- */ MPIDI_Progress_signal(); }
void MPID_Request_set_completed(MPID_Request *req) { MPIR_cc_set(&req->cc, 0); MPIDI_Progress_signal(); }
/** * \brief The callback for a new RZV RTS * \note Because this is a short message, the data is already received * \param[in] context The context on which the message is being received. * \param[in] sender The origin endpoint * \param[in] _msginfo The extended header information * \param[in] msginfo_size The size of the extended header information * \param[in] is_zero_byte The rendezvous message is zero bytes in length. */ void MPIDI_RecvRzvCB_impl(pami_context_t context, pami_endpoint_t sender, const void * _msginfo, size_t msginfo_size, const unsigned is_zero_byte) { MPID_assert(_msginfo != NULL); MPID_assert(msginfo_size == sizeof(MPIDI_MsgEnvelope)); const MPIDI_MsgEnvelope * envelope = (const MPIDI_MsgEnvelope *)_msginfo; const MPIDI_MsgInfo * msginfo = (const MPIDI_MsgInfo *)&envelope->msginfo; MPID_Request * rreq = NULL; int found; pami_task_t source; #if TOKEN_FLOW_CONTROL int rettoks=0; #endif /* -------------------- */ /* Match the request. */ /* -------------------- */ unsigned rank = msginfo->MPIrank; unsigned tag = msginfo->MPItag; unsigned context_id = msginfo->MPIctxt; MPID_Request *newreq = MPIDI_Request_create2(); MPIU_THREAD_CS_ENTER(MSGQUEUE,0); source = PAMIX_Endpoint_query(sender); MPIDI_Receive_tokens(msginfo,source); #ifndef OUT_OF_ORDER_HANDLING rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, tag, context_id, &found); #else rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, source, tag, context_id, msginfo->MPIseqno, &found); #endif TRACE_ERR("RZV CB for req=%p remote-mr=0x%llx bytes=%zu (%sfound)\n", rreq, *(unsigned long long*)&envelope->envelope.memregion, envelope->envelope.length, found?"":"not "); /* ---------------------- */ /* Copy in information. */ /* ---------------------- */ rreq->status.MPI_SOURCE = rank; rreq->status.MPI_TAG = tag; MPIR_STATUS_SET_COUNT(rreq->status, envelope->length); MPIDI_Request_setPeerRank_comm(rreq, rank); MPIDI_Request_setPeerRank_pami(rreq, source); MPIDI_Request_cpyPeerRequestH (rreq, msginfo); MPIDI_Request_setSync (rreq, msginfo->isSync); MPIDI_Request_setRzv (rreq, 1); /* ----------------------------------------------------- */ /* Save the rendezvous information for when the target */ /* node calls a receive function and the data is */ /* retreived from the origin node. */ /* ----------------------------------------------------- */ if (is_zero_byte) { rreq->mpid.envelope.length = 0; rreq->mpid.envelope.data = NULL; } else { #ifdef USE_PAMI_RDMA memcpy(&rreq->mpid.envelope.memregion, &envelope->memregion, sizeof(pami_memregion_t)); #else rreq->mpid.envelope.memregion_used = envelope->memregion_used; if(envelope->memregion_used) { memcpy(&rreq->mpid.envelope.memregion, &envelope->memregion, sizeof(pami_memregion_t)); } rreq->mpid.envelope.data = envelope->data; #endif rreq->mpid.envelope.length = envelope->length; TRACE_SET_R_VAL(source,(rreq->mpid.idx),req,rreq); TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,envelope->length); TRACE_SET_R_VAL(source,(rreq->mpid.idx),fl.f.sync,msginfo->isSync); TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.rzv); if (TOKEN_FLOW_CONTROL_ON) { #if TOKEN_FLOW_CONTROL MPIDI_Must_return_tokens(context,source); #else MPID_assert_always(0); #endif } } /* ----------------------------------------- */ /* figure out target buffer for request data */ /* ----------------------------------------- */ if (found) { #if (MPIDI_STATISTICS) MPID_NSTAT(mpid_statp->earlyArrivalsMatched); #endif /* --------------------------- */ /* if synchronized, post ack. */ /* --------------------------- */ if (unlikely(MPIDI_Request_isSync(rreq))) MPIDI_SyncAck_post(context, rreq, MPIDI_Request_getPeerRank_pami(rreq)); MPIU_THREAD_CS_EXIT(MSGQUEUE,0); if (is_zero_byte) MPIDI_RecvRzvDoneCB_zerobyte(context, rreq, PAMI_SUCCESS); else { MPIDI_RendezvousTransfer(context, rreq); TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.sync_com_in_HH); TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.matchedInHH); TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf); } MPID_Request_discard(newreq); } /* ------------------------------------------------------------- */ /* Request was not posted. */ /* ------------------------------------------------------------- */ else { #if (MPIDI_STATISTICS) MPID_NSTAT(mpid_statp->earlyArrivals); #endif /* * This is to test that the fields don't need to be * initialized. Remove after this doesn't fail for a while. */ MPID_assert(rreq->mpid.uebuf == NULL); MPID_assert(rreq->mpid.uebuflen == 0); /* rreq->mpid.uebuf = NULL; */ /* rreq->mpid.uebuflen = 0; */ #ifdef OUT_OF_ORDER_HANDLING if (MPIDI_In_cntr[source].n_OutOfOrderMsgs > 0) { MPIDI_Recvq_process_out_of_order_msgs(source, context); } #endif MPIU_THREAD_CS_EXIT(MSGQUEUE,0); } /* ---------------------------------------- */ /* Signal that the recv has been started. */ /* ---------------------------------------- */ MPIDI_Progress_signal(); }