static void cb_barrier(void *ctxt, void *clientdata, pami_result_t err)
{
int *active = (int *) clientdata;
TRACE_ERR("callback. enter: %d\n", (*active));
MPIDI_Progress_signal();
(*active)--;
}
void
MPIDI_Win_DoneCB(pami_context_t context,
void * cookie,
pami_result_t result)
{
MPIDI_Win_request *req = (MPIDI_Win_request*)cookie;
++req->win->mpid.sync.complete;
if ((req->buffer_free) && (req->type == MPIDI_WIN_REQUEST_GET))
{
++req->origin.completed;
if (req->origin.completed == req->target.dt.num_contig)
{
int mpi_errno;
mpi_errno = MPIR_Localcopy(req->buffer,
req->origin.dt.size,
MPI_CHAR,
req->origin.addr,
req->origin.count,
req->origin.datatype);
MPID_assert(mpi_errno == MPI_SUCCESS);
MPID_Datatype_release(req->origin.dt.pointer);
MPIU_Free(req->buffer);
req->buffer_free = 0;
}
}
if (req->win->mpid.sync.total == req->win->mpid.sync.complete)
{
if (req->buffer_free)
MPIU_Free(req->buffer);
if (req->accum_headers)
MPIU_Free(req->accum_headers);
MPIU_Free(req);
}
MPIDI_Progress_signal();
}
static inline void
MPIDI_RecvShortCB(pami_context_t context,
const void * _msginfo,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
unsigned isSync)
{
MPID_assert(_msginfo != NULL);
const MPIDI_MsgInfo *msginfo = (const MPIDI_MsgInfo *)_msginfo;
MPID_Request * rreq = NULL;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, source, tag, context_id, msginfo->MPIseqno);
#endif
/* Match not found */
if (unlikely(rreq == NULL))
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request *newreq = MPIDI_Request_create2();
MPID_assert(newreq != NULL);
if (sndlen)
{
newreq->mpid.uebuflen = sndlen;
if (!TOKEN_FLOW_CONTROL_ON)
{
newreq->mpid.uebuf = MPL_malloc(sndlen);
newreq->mpid.uebuf_malloc = mpiuMalloc;
}
else
{
#if TOKEN_FLOW_CONTROL
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
newreq->mpid.uebuf = MPIDI_mm_alloc(sndlen);
newreq->mpid.uebuf_malloc = mpidiBufMM;
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#else
MPID_assert_always(0);
#endif
}
MPID_assert(newreq->mpid.uebuf != NULL);
}
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, PAMIX_Endpoint_query(sender), tag, context_id, msginfo->MPIseqno);
#endif
if (unlikely(rreq == NULL))
{
MPIDI_Callback_process_unexp(newreq, context, msginfo, sndlen, sender, sndbuf, NULL, isSync);
/* request is always complete now */
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Token_cntr[source].unmatched++;
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_release(newreq);
goto fn_exit_short;
}
else
{
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_discard(newreq);
}
}
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Update_rettoks(source);
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* the receive queue processing has been completed and we found match*/
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, sndlen);
MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE);
MPIDI_Request_cpyPeerRequestH(rreq, msginfo);
MPIDI_Request_setSync (rreq, isSync);
MPIDI_Request_setRzv (rreq, 0);
/* ----------------------------- */
/* Request was already posted. */
/* ----------------------------- */
if (unlikely(isSync))
MPIDI_SyncAck_post(context, rreq, PAMIX_Endpoint_query(sender));
if (unlikely(HANDLE_GET_KIND(rreq->mpid.datatype) != HANDLE_KIND_BUILTIN))
{
MPIDI_Callback_process_userdefined_dt(context, sndbuf, sndlen, rreq);
goto fn_exit_short;
}
size_t dt_size = rreq->mpid.userbufcount * MPID_Datatype_get_basic_size(rreq->mpid.datatype);
/* ----------------------------- */
/* Test for truncated message. */
/* ----------------------------- */
if (unlikely(sndlen > dt_size))
{
#if ASSERT_LEVEL > 0
MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf);
goto fn_exit_short;
#else
sndlen = dt_size;
#endif
}
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
void* rcvbuf = rreq->mpid.userbuf;
if (sndlen > 0)
{
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf))
{
cudaError_t cudaerr = CudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice);
}
else
#endif
memcpy(rcvbuf, sndbuf, sndlen);
}
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
MPIDI_Request_complete(rreq);
fn_exit_short:
#ifdef OUT_OF_ORDER_HANDLING
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs>0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#endif
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
void
MPID_Request_set_completed(MPID_Request *req)
{
MPIR_cc_set(&req->cc, 0);
MPIDI_Progress_signal();
}
/**
* \brief The callback for a new RZV RTS
* \note Because this is a short message, the data is already received
* \param[in] context The context on which the message is being received.
* \param[in] sender The origin endpoint
* \param[in] _msginfo The extended header information
* \param[in] msginfo_size The size of the extended header information
* \param[in] is_zero_byte The rendezvous message is zero bytes in length.
*/
void
MPIDI_RecvRzvCB_impl(pami_context_t context,
pami_endpoint_t sender,
const void * _msginfo,
size_t msginfo_size,
const unsigned is_zero_byte)
{
MPID_assert(_msginfo != NULL);
MPID_assert(msginfo_size == sizeof(MPIDI_MsgEnvelope));
const MPIDI_MsgEnvelope * envelope = (const MPIDI_MsgEnvelope *)_msginfo;
const MPIDI_MsgInfo * msginfo = (const MPIDI_MsgInfo *)&envelope->msginfo;
MPID_Request * rreq = NULL;
int found;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPID_Request *newreq = MPIDI_Request_create2();
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, tag, context_id, &found);
#else
rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, source, tag, context_id, msginfo->MPIseqno, &found);
#endif
TRACE_ERR("RZV CB for req=%p remote-mr=0x%llx bytes=%zu (%sfound)\n",
rreq,
*(unsigned long long*)&envelope->envelope.memregion,
envelope->envelope.length,
found?"":"not ");
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, envelope->length);
MPIDI_Request_setPeerRank_comm(rreq, rank);
MPIDI_Request_setPeerRank_pami(rreq, source);
MPIDI_Request_cpyPeerRequestH (rreq, msginfo);
MPIDI_Request_setSync (rreq, msginfo->isSync);
MPIDI_Request_setRzv (rreq, 1);
/* ----------------------------------------------------- */
/* Save the rendezvous information for when the target */
/* node calls a receive function and the data is */
/* retreived from the origin node. */
/* ----------------------------------------------------- */
if (is_zero_byte)
{
rreq->mpid.envelope.length = 0;
rreq->mpid.envelope.data = NULL;
}
else
{
#ifdef USE_PAMI_RDMA
memcpy(&rreq->mpid.envelope.memregion,
&envelope->memregion,
sizeof(pami_memregion_t));
#else
rreq->mpid.envelope.memregion_used = envelope->memregion_used;
if(envelope->memregion_used)
{
memcpy(&rreq->mpid.envelope.memregion,
&envelope->memregion,
sizeof(pami_memregion_t));
}
rreq->mpid.envelope.data = envelope->data;
#endif
rreq->mpid.envelope.length = envelope->length;
TRACE_SET_R_VAL(source,(rreq->mpid.idx),req,rreq);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,envelope->length);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),fl.f.sync,msginfo->isSync);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.rzv);
if (TOKEN_FLOW_CONTROL_ON)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
}
/* ----------------------------------------- */
/* figure out target buffer for request data */
/* ----------------------------------------- */
if (found)
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
/* --------------------------- */
/* if synchronized, post ack. */
/* --------------------------- */
if (unlikely(MPIDI_Request_isSync(rreq)))
MPIDI_SyncAck_post(context, rreq, MPIDI_Request_getPeerRank_pami(rreq));
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
if (is_zero_byte)
MPIDI_RecvRzvDoneCB_zerobyte(context, rreq, PAMI_SUCCESS);
else
{
MPIDI_RendezvousTransfer(context, rreq);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.sync_com_in_HH);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.matchedInHH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
}
MPID_Request_discard(newreq);
}
/* ------------------------------------------------------------- */
/* Request was not posted. */
/* ------------------------------------------------------------- */
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
/*
* This is to test that the fields don't need to be
* initialized. Remove after this doesn't fail for a while.
*/
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
/* rreq->mpid.uebuf = NULL; */
/* rreq->mpid.uebuflen = 0; */
#ifdef OUT_OF_ORDER_HANDLING
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs > 0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
