int MPID_nem_tcp_ckpt_restart_vc(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
MPIDI_nem_tcp_pkt_unpause_t * const pkt = (MPIDI_nem_tcp_pkt_unpause_t *)&upkt;
MPIR_Request *sreq;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_TCP_CKPT_RESTART_VC);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_TCP_CKPT_RESTART_VC);
pkt->type = MPIDI_NEM_PKT_NETMOD;
pkt->subtype = MPIDI_NEM_TCP_PKT_UNPAUSE;
mpi_errno = MPID_nem_tcp_iStartContigMsg_paused(vc, pkt, sizeof(pkt), NULL, 0, &sreq);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (sreq != NULL) {
if (sreq->status.MPI_ERROR != MPI_SUCCESS) {
mpi_errno = sreq->status.MPI_ERROR;
MPIR_Request_free(sreq);
MPIR_ERR_INTERNALANDJUMP(mpi_errno, "Failed to send checkpoint unpause pkt.");
}
MPIR_Request_free(sreq);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_TCP_CKPT_RESTART_VC);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_RecvRndv( MPIDI_VC_t * vc, MPIR_Request *rreq )
{
int mpi_errno = MPI_SUCCESS;
/* A rendezvous request-to-send (RTS) message has arrived. We need
to send a CTS message to the remote process. */
MPIR_Request * cts_req;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_rndv_clr_to_send_t * cts_pkt = &upkt.rndv_clr_to_send;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,
"rndv RTS in the request, sending rndv CTS");
MPIDI_Pkt_init(cts_pkt, MPIDI_CH3_PKT_RNDV_CLR_TO_SEND);
cts_pkt->sender_req_id = rreq->dev.sender_req_id;
cts_pkt->receiver_req_id = rreq->handle;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, cts_pkt, sizeof(*cts_pkt), &cts_req);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**ch3|ctspkt");
}
if (cts_req != NULL)
{
/* FIXME: Ideally we could specify that a req not be returned.
This would avoid our having to decrement the
reference count on a req we don't want/need. */
MPIR_Request_free(cts_req);
}
fn_fail:
return mpi_errno;
}
int MPID_nem_tcp_ckpt_continue_vc(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPID_PKT_DECL_CAST(upkt, MPIDI_nem_tcp_pkt_unpause_t, unpause_pkt);
MPIR_Request *unpause_req;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_TCP_CKPT_CONTINUE_VC);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_TCP_CKPT_CONTINUE_VC);
unpause_pkt->type = MPIDI_NEM_PKT_NETMOD;
unpause_pkt->subtype = MPIDI_NEM_TCP_PKT_UNPAUSE;
mpi_errno = MPID_nem_tcp_iStartContigMsg_paused(vc, &upkt, sizeof(MPIDI_nem_tcp_pkt_unpause_t), NULL, 0, &unpause_req);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (unpause_req) {
if (unpause_req->status.MPI_ERROR) MPIR_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**fail");
MPIR_Request_free(unpause_req);
if (mpi_errno) goto fn_fail;
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_TCP_CKPT_CONTINUE_VC);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPIDI_CH3U_VC_SendClose - Initiate a close on a virtual connection
Input Parameters:
+ vc - Virtual connection to close
- i - rank of virtual connection within a process group (used for debugging)
Notes:
The current state of this connection must be either 'MPIDI_VC_STATE_ACTIVE'
or 'MPIDI_VC_STATE_REMOTE_CLOSE'.
@*/
int MPIDI_CH3U_VC_SendClose( MPIDI_VC_t *vc, int rank )
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_close_t * close_pkt = &upkt.close;
MPIR_Request * sreq;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_VC_SENDCLOSE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_VC_SENDCLOSE);
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
MPIR_Assert( vc->state == MPIDI_VC_STATE_ACTIVE ||
vc->state == MPIDI_VC_STATE_REMOTE_CLOSE );
MPIDI_Pkt_init(close_pkt, MPIDI_CH3_PKT_CLOSE);
close_pkt->ack = (vc->state == MPIDI_VC_STATE_ACTIVE) ? FALSE : TRUE;
/* MT: this is not thread safe, the POBJ CS is scoped to the vc and
* doesn't protect this global correctly */
MPIDI_Outstanding_close_ops += 1;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_DISCONNECT,TYPICAL,(MPL_DBG_FDEST,
"sending close(%s) on vc (pg=%p) %p to rank %d, ops = %d",
close_pkt->ack ? "TRUE" : "FALSE", vc->pg, vc,
rank, MPIDI_Outstanding_close_ops));
/*
* A close packet acknowledging this close request could be
* received during iStartMsg, therefore the state must
* be changed before the close packet is sent.
*/
if (vc->state == MPIDI_VC_STATE_ACTIVE) {
MPIDI_CHANGE_VC_STATE(vc, LOCAL_CLOSE);
}
else {
MPIR_Assert( vc->state == MPIDI_VC_STATE_REMOTE_CLOSE );
MPIDI_CHANGE_VC_STATE(vc, CLOSE_ACKED);
}
mpi_errno = MPIDI_CH3_iStartMsg(vc, close_pkt, sizeof(*close_pkt), &sreq);
MPIR_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|send_close_ack");
if (sreq != NULL) {
/* There is still another reference being held by the channel. It
will not be released until the pkt is actually sent. */
MPIR_Request_free(sreq);
}
fn_exit:
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_VC_SENDCLOSE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* Send a zero-sized message with eager synchronous. This is a temporary
routine, as we may want to replace this with a counterpart to the
Eager Short message */
int MPIDI_CH3_EagerSyncZero(MPIR_Request **sreq_p, int rank, int tag,
MPIR_Comm * comm, int context_offset )
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_sync_send_t * const es_pkt = &upkt.eager_sync_send;
MPIDI_VC_t * vc;
MPIR_Request *sreq = *sreq_p;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"sending zero length message");
/* MT FIXME what are the two operations we are waiting for? the send and
* the sync response? */
MPIR_cc_set(&sreq->cc, 2);
MPIDI_Request_set_msg_type(sreq, MPIDI_REQUEST_EAGER_MSG);
sreq->dev.OnDataAvail = 0;
MPIDI_Pkt_init(es_pkt, MPIDI_CH3_PKT_EAGER_SYNC_SEND);
es_pkt->match.parts.rank = comm->rank;
es_pkt->match.parts.tag = tag;
es_pkt->match.parts.context_id = comm->context_id + context_offset;
es_pkt->sender_req_id = sreq->handle;
es_pkt->data_sz = 0;
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(es_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPL_DBG_MSGPKT(vc,tag,es_pkt->match.parts.context_id,rank,(intptr_t)0,"EagerSync0");
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iSend(vc, sreq, es_pkt, sizeof(*es_pkt));
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIR_Request_free(sreq);
*sreq_p = NULL;
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static
void rma_put_done(pscom_request_t *req)
{
MPIR_Request *mpid_req = req->user->type.put_send.mpid_req;
assert(pscom_req_successful(req));
/* This is an pscom.io_done call. Global lock state undefined! */
MPID_PSP_packed_msg_cleanup(&req->user->type.put_send.msg);
/* ToDo: this is not threadsave */
req->user->type.put_send.win_ptr->rma_local_pending_cnt--;
req->user->type.put_send.win_ptr->rma_local_pending_rank[req->user->type.put_send.target_rank]--;
if(mpid_req) {
MPID_PSP_Subrequest_completed(mpid_req);
MPIR_Request_free(mpid_req);
} else {
pscom_request_free(req);
}
}
static inline int request_complete_fastpath(MPI_Request *request, MPIR_Request *request_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Assert(request_ptr->kind == MPIR_REQUEST_KIND__SEND || request_ptr->kind == MPIR_REQUEST_KIND__RECV);
if (request_ptr->kind == MPIR_REQUEST_KIND__SEND) {
/* FIXME: are Ibsend requests added to the send queue? */
MPII_SENDQ_FORGET(request_ptr);
}
/* the completion path for SEND and RECV is the same at this time, modulo
* the SENDQ hook above */
mpi_errno = request_ptr->status.MPI_ERROR;
MPIR_Request_free(request_ptr);
*request = MPI_REQUEST_NULL;
/* avoid normal fn_exit/fn_fail jump pattern to reduce jumps and compiler confusion */
return mpi_errno;
}
int MPIR_Test(MPI_Request * request, int *flag, MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
int active_flag;
MPIR_Request *request_ptr = NULL;
/* If this is a null request handle, then return an empty status */
if (*request == MPI_REQUEST_NULL) {
MPIR_Status_set_empty(status);
*flag = TRUE;
goto fn_exit;
}
MPIR_Request_get_ptr(*request, request_ptr);
MPIR_Assert(request_ptr != NULL);
mpi_errno = MPID_Test(request_ptr, flag, status);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
if (*flag) {
mpi_errno = MPIR_Request_completion_processing(request_ptr, status, &active_flag);
if (!MPIR_Request_is_persistent(request_ptr)) {
MPIR_Request_free(request_ptr);
*request = MPI_REQUEST_NULL;
}
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Fall through to the exit */
} else if (unlikely(MPIR_Request_is_anysrc_mismatched(request_ptr))) {
MPIR_ERR_SET(mpi_errno, MPIX_ERR_PROC_FAILED_PENDING, "**failure_pending");
if (status != MPI_STATUS_IGNORE)
status->MPI_ERROR = mpi_errno;
goto fn_fail;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*
* These routines are called when a receive matches an eager sync send
*/
int MPIDI_CH3_EagerSyncAck( MPIDI_VC_t *vc, MPIR_Request *rreq )
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_sync_ack_t * const esa_pkt = &upkt.eager_sync_ack;
MPIR_Request * esa_req;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"sending eager sync ack");
MPIDI_Pkt_init(esa_pkt, MPIDI_CH3_PKT_EAGER_SYNC_ACK);
esa_pkt->sender_req_id = rreq->dev.sender_req_id;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, esa_pkt, sizeof(*esa_pkt), &esa_req);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
if (esa_req != NULL)
{
MPIR_Request_free(esa_req);
}
fn_fail:
return mpi_errno;
}
int MPID_Cancel_send(MPIR_Request * sreq)
{
MPIDI_VC_t * vc;
int proto;
int flag;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_CANCEL_SEND);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_CANCEL_SEND);
MPIR_Assert(sreq->kind == MPIR_REQUEST_KIND__SEND);
MPIDI_Request_cancel_pending(sreq, &flag);
if (flag)
{
goto fn_exit;
}
/*
* FIXME: user requests returned by MPI_Ibsend() have a NULL comm pointer
* and no pointer to the underlying communication
* request. For now, we simply fail to cancel the request. In the future,
* we should add a new request kind to indicate that
* the request is a BSEND. Then we can properly cancel the request, much
* in the way we do persistent requests.
*/
if (sreq->comm == NULL)
{
goto fn_exit;
}
MPIDI_Comm_get_vc_set_active(sreq->comm, sreq->dev.match.parts.rank, &vc);
proto = MPIDI_Request_get_msg_type(sreq);
if (proto == MPIDI_REQUEST_SELF_MSG)
{
MPIR_Request * rreq;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,
"attempting to cancel message sent to self");
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
rreq = MPIDI_CH3U_Recvq_FDU(sreq->handle, &sreq->dev.match);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
if (rreq)
{
MPIR_Assert(rreq->dev.partner_request == sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"send-to-self cancellation successful, sreq=0x%08x, rreq=0x%08x",
sreq->handle, rreq->handle));
/* Pull the message out of the unexpected queue since it's
* being cancelled. The below request release drops one
* reference. We explicitly drop a second reference,
* because the receive request will never be visible to
* the user. */
MPIR_Request_free(rreq);
MPIR_Request_free(rreq);
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else
{
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"send-to-self cancellation failed, sreq=0x%08x, rreq=0x%08x",
sreq->handle, rreq->handle));
}
goto fn_exit;
}
/* If the message went over a netmod and it provides a cancel_send
function, call it here. */
#ifdef ENABLE_COMM_OVERRIDES
if (vc->comm_ops && vc->comm_ops->cancel_send)
{
mpi_errno = vc->comm_ops->cancel_send(vc, sreq);
goto fn_exit;
}
#endif
/* Check to see if the send is still in the send queue. If so, remove it,
mark the request and cancelled and complete, and
release the device's reference to the request object.
*/
{
int cancelled;
if (proto == MPIDI_REQUEST_RNDV_MSG)
{
MPIR_Request * rts_sreq;
/* The cancellation of the RTS request needs to be atomic through
the destruction of the RTS request to avoid
conflict with release of the RTS request if the CTS is received
(see handling of a rendezvous CTS packet in
MPIDI_CH3U_Handle_recv_pkt()).
MPID_Request_fetch_and_clear_rts_sreq() is used to gurantee
that atomicity. */
MPIDI_Request_fetch_and_clear_rts_sreq(sreq, &rts_sreq);
if (rts_sreq != NULL)
{
cancelled = FALSE;
/* since we attempted to cancel a RTS request, then we are
responsible for releasing that request */
MPIR_Request_free(rts_sreq);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, __func__, __LINE__, MPI_ERR_OTHER,
"**ch3|cancelrndv", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
if (cancelled)
{
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* no other thread should be waiting on sreq, so it is
safe to reset ref_count and cc */
MPIR_cc_set(&sreq->cc, 0);
/* FIXME should be a decr and assert, not a set */
MPIR_Object_set_ref(sreq, 1);
goto fn_exit;
}
}
}
else
{
cancelled = FALSE;
if (cancelled)
{
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* no other thread should be waiting on sreq, so it is safe to
reset ref_count and cc */
MPIR_cc_set(&sreq->cc, 0);
/* FIXME should be a decr and assert, not a set */
MPIR_Object_set_ref(sreq, 1);
goto fn_exit;
}
}
}
/* Part or all of the message has already been sent, so we need to send a
cancellation request to the receiver in an attempt
to catch the message before it is matched. */
{
int was_incomplete;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_cancel_send_req_t * const csr_pkt = &upkt.cancel_send_req;
MPIR_Request * csr_sreq;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"sending cancel request to %d for 0x%08x",
sreq->dev.match.parts.rank, sreq->handle));
/* The completion counter and reference count are incremented to keep
the request around long enough to receive a
response regardless of what the user does (free the request before
waiting, etc.). */
MPIDI_CH3U_Request_increment_cc(sreq, &was_incomplete);
if (!was_incomplete)
{
/* The reference count is incremented only if the request was
complete before the increment. */
MPIR_Request_add_ref( sreq );
}
MPIDI_Pkt_init(csr_pkt, MPIDI_CH3_PKT_CANCEL_SEND_REQ);
csr_pkt->match.parts.rank = sreq->comm->rank;
csr_pkt->match.parts.tag = sreq->dev.match.parts.tag;
csr_pkt->match.parts.context_id = sreq->dev.match.parts.context_id;
csr_pkt->sender_req_id = sreq->handle;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, csr_pkt, sizeof(*csr_pkt), &csr_sreq);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|cancelreq");
}
if (csr_sreq != NULL)
{
MPIR_Request_free(csr_sreq);
}
}
/* FIXME: if send cancellation packets are allowed to arrive out-of-order
with respect to send packets, then we need to
timestamp send and cancel packets to insure that a cancellation request
does not bypass the send packet to be cancelled
and erroneously cancel a previously sent message with the same request
handle. */
/* FIXME: A timestamp is more than is necessary; a message sequence number
should be adequate. */
fn_fail:
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_CANCEL_SEND);
return mpi_errno;
}
int MPID_nem_ptl_improbe(MPIDI_VC_t *vc, int source, int tag, MPIR_Comm *comm, int context_offset, int *flag,
MPIR_Request **message, MPI_Status *status)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
int ret;
ptl_process_t id_any;
ptl_me_t me;
MPIR_Request *req;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_PTL_IMPROBE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_PTL_IMPROBE);
id_any.phys.nid = PTL_NID_ANY;
id_any.phys.pid = PTL_PID_ANY;
/* create a request */
req = MPIR_Request_create(MPIR_REQUEST_KIND__MPROBE);
MPID_nem_ptl_init_req(req);
MPIR_ERR_CHKANDJUMP1(!req, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIR_Request_create");
MPIR_Object_set_ref(req, 2); /* 1 ref for progress engine and 1 ref for us */
REQ_PTL(req)->event_handler = handle_mprobe;
/* create a dummy ME to use for searching the list */
me.start = NULL;
me.length = 0;
me.ct_handle = PTL_CT_NONE;
me.uid = PTL_UID_ANY;
me.options = ( PTL_ME_OP_PUT | PTL_ME_USE_ONCE );
me.min_free = 0;
me.match_bits = NPTL_MATCH(tag, comm->context_id + context_offset, source);
if (source == MPI_ANY_SOURCE)
me.match_id = id_any;
else {
if (!vc_ptl->id_initialized) {
mpi_errno = MPID_nem_ptl_init_id(vc);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
me.match_id = vc_ptl->id;
}
if (tag == MPI_ANY_TAG)
me.ignore_bits = NPTL_MATCH_IGNORE_ANY_TAG;
else
me.ignore_bits = NPTL_MATCH_IGNORE;
/* submit a search request */
ret = PtlMESearch(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_pt, &me, PTL_SEARCH_DELETE, req);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmesearch", "**ptlmesearch %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MESearch("REG", vc ? vc->pg_rank : 0, me, req);
/* wait for search request to complete */
do {
mpi_errno = MPID_nem_ptl_poll(FALSE);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
} while (!MPIR_Request_is_complete(req));
*flag = REQ_PTL(req)->found;
if (*flag) {
req->comm = comm;
MPIR_Comm_add_ref(comm);
MPIR_Request_extract_status(req, status);
*message = req;
}
else {
MPIR_Request_free(req);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_PTL_IMPROBE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int ADD_SUFFIX(MPID_nem_ofi_iprobe_impl)(struct MPIDI_VC *vc,
int source,
int tag,
MPIR_Comm * comm,
int context_offset,
int *flag, MPI_Status * status, MPIR_Request ** rreq_ptr)
{
int ret, mpi_errno = MPI_SUCCESS;
fi_addr_t remote_proc = 0;
uint64_t match_bits, mask_bits;
size_t len;
MPIR_Request rreq_s, *rreq;
BEGIN_FUNC(FCNAME);
if (rreq_ptr) {
MPIDI_CH3I_NM_OFI_RC(MPID_nem_ofi_create_req(&rreq, 1));
rreq->kind = MPIR_REQUEST_KIND__RECV;
*rreq_ptr = rreq;
rreq->comm = comm;
rreq->dev.match.parts.rank = source;
rreq->dev.match.parts.tag = tag;
rreq->dev.match.parts.context_id = comm->context_id;
MPIR_Comm_add_ref(comm);
}
else {
rreq = &rreq_s;
rreq->dev.OnDataAvail = NULL;
}
REQ_OFI(rreq)->pack_buffer = NULL;
REQ_OFI(rreq)->event_callback = ADD_SUFFIX(peek_callback);
REQ_OFI(rreq)->match_state = PEEK_INIT;
OFI_ADDR_INIT(source, vc, remote_proc);
#if API_SET == API_SET_1
match_bits = init_recvtag(&mask_bits, comm->context_id + context_offset, source, tag);
#elif API_SET == API_SET_2
match_bits = init_recvtag_2(&mask_bits, comm->context_id + context_offset, tag);
#endif
/* ------------------------------------------------------------------------- */
/* fi_recvmsg with FI_PEEK: */
/* Initiate a search for a match in the hardware or software queue. */
/* The search can complete immediately with -ENOMSG. */
/* I successful, libfabric will enqueue a context entry into the completion */
/* queue to make the search nonblocking. This code will poll until the */
/* entry is enqueued. */
/* ------------------------------------------------------------------------- */
msg_tagged_t msg;
uint64_t msgflags = FI_PEEK;
msg.msg_iov = NULL;
msg.desc = NULL;
msg.iov_count = 0;
msg.addr = remote_proc;
msg.tag = match_bits;
msg.ignore = mask_bits;
msg.context = (void *) &(REQ_OFI(rreq)->ofi_context);
msg.data = 0;
if(*flag == CLAIM_PEEK)
msgflags|=FI_CLAIM;
ret = fi_trecvmsg(gl_data.endpoint,&msg,msgflags);
if(ret == -ENOMSG) {
if (rreq_ptr) {
MPIR_Request_free(rreq);
*rreq_ptr = NULL;
*flag = 0;
}
MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL);
goto fn_exit;
}
MPIR_ERR_CHKANDJUMP4((ret < 0), mpi_errno, MPI_ERR_OTHER,
"**ofi_peek", "**ofi_peek %s %d %s %s",
__SHORT_FILE__, __LINE__, FCNAME, fi_strerror(-ret));
while (PEEK_INIT == REQ_OFI(rreq)->match_state)
MPID_nem_ofi_poll(MPID_BLOCKING_POLL);
if (PEEK_NOT_FOUND == REQ_OFI(rreq)->match_state) {
if (rreq_ptr) {
MPIR_Request_free(rreq);
*rreq_ptr = NULL;
*flag = 0;
}
MPID_nem_ofi_poll(MPID_NONBLOCKING_POLL);
goto fn_exit;
}
if (status != MPI_STATUS_IGNORE)
*status = rreq->status;
MPIR_Request_add_ref(rreq);
*flag = 1;
END_FUNC_RC(FCNAME);
}
/* MPIDI_CH3_RndvSend - Send a request to perform a rendezvous send */
int MPIDI_CH3_RndvSend( MPIR_Request **sreq_p, const void * buf, MPI_Aint count,
MPI_Datatype datatype, int dt_contig, intptr_t data_sz,
MPI_Aint dt_true_lb,
int rank,
int tag, MPIR_Comm * comm, int context_offset )
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_rndv_req_to_send_t * const rts_pkt = &upkt.rndv_req_to_send;
MPIDI_VC_t * vc;
MPIR_Request * rts_sreq;
MPIR_Request *sreq =*sreq_p;
int mpi_errno = MPI_SUCCESS;
MPL_DBG_MSG_D(MPIDI_CH3_DBG_OTHER,VERBOSE,
"sending rndv RTS, data_sz=%" PRIdPTR, data_sz);
sreq->dev.OnDataAvail = 0;
sreq->dev.partner_request = NULL;
MPIDI_Pkt_init(rts_pkt, MPIDI_CH3_PKT_RNDV_REQ_TO_SEND);
rts_pkt->match.parts.rank = comm->rank;
rts_pkt->match.parts.tag = tag;
rts_pkt->match.parts.context_id = comm->context_id + context_offset;
rts_pkt->sender_req_id = sreq->handle;
rts_pkt->data_sz = data_sz;
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(rts_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPL_DBG_MSGPKT(vc,tag,rts_pkt->match.parts.context_id,rank,data_sz,"Rndv");
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, rts_pkt, sizeof(*rts_pkt), &rts_sreq);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIR_Request_free(sreq);
*sreq_p = NULL;
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rtspkt");
}
/* --END ERROR HANDLING-- */
if (rts_sreq != NULL)
{
if (rts_sreq->status.MPI_ERROR != MPI_SUCCESS)
{
MPIR_Request_free(sreq);
*sreq_p = NULL;
mpi_errno = rts_sreq->status.MPI_ERROR;
MPIR_Request_free(rts_sreq);
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rtspkt");
}
MPIR_Request_free(rts_sreq);
}
/* FIXME: fill temporary IOV or pack temporary buffer after send to hide
some latency. This requires synchronization
because the CTS packet could arrive and be processed before the above
iStartmsg completes (depending on the progress
engine, threads, etc.). */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Put_generic(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype,
int target_rank, MPI_Aint target_disp, int target_count,
MPI_Datatype target_datatype, MPIR_Win *win_ptr, MPIR_Request **request)
{
int mpi_error = MPI_SUCCESS;
MPID_PSP_Datatype_info dt_info;
MPID_PSP_packed_msg_t msg;
MPID_Win_rank_info *ri = win_ptr->rank_info + target_rank;
char *target_buf;
#if 0
fprintf(stderr, "int MPID_Put(origin_addr: %p, origin_count: %d, origin_datatype: %08x,"
" target_rank: %d, target_disp: %d, target_count: %d, target_datatype: %08x,"
" *win_ptr: %p)\n",
origin_addr, origin_count, origin_datatype,
target_rank, target_disp, target_count,
target_datatype, win_ptr);
#endif
/* Datatype */
MPID_PSP_Datatype_get_info(target_datatype, &dt_info);
if(request) {
*request = MPIR_Request_create(MPIR_REQUEST_KIND__SEND);
(*request)->comm = win_ptr->comm_ptr;
MPIR_Comm_add_ref(win_ptr->comm_ptr);
}
if (unlikely(target_rank == MPI_PROC_NULL)) {
goto fn_completed;
}
/* Request-based RMA operations are only valid within a passive target epoch! */
if(request && win_ptr->epoch_state != MPID_PSP_EPOCH_LOCK && win_ptr->epoch_state != MPID_PSP_EPOCH_LOCK_ALL) {
mpi_error = MPI_ERR_RMA_SYNC;
goto err_sync_rma;
}
/* Check that we are within an access/exposure epoch: */
if (win_ptr->epoch_state == MPID_PSP_EPOCH_NONE) {
mpi_error = MPI_ERR_RMA_SYNC;
goto err_sync_rma;
}
/* Track access epoch state: */
if (win_ptr->epoch_state == MPID_PSP_EPOCH_FENCE_ISSUED) {
win_ptr->epoch_state = MPID_PSP_EPOCH_FENCE;
}
/* If the put is a local operation, do it here */
if (target_rank == win_ptr->rank || win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) {
void *base;
int disp_unit;
if (win_ptr->create_flavor == MPI_WIN_FLAVOR_SHARED) {
MPID_PSP_shm_rma_get_base(win_ptr, target_rank, &disp_unit, &base);
}
else {
base = win_ptr->base;
disp_unit = win_ptr->disp_unit;
}
mpi_error = MPIR_Localcopy(origin_addr, origin_count, origin_datatype,
(char *) base + disp_unit * target_disp,
target_count, target_datatype);
if (mpi_error) {
goto err_local_copy;
}
goto fn_completed;
}
/* Data */
mpi_error = MPID_PSP_packed_msg_prepare(origin_addr, origin_count, origin_datatype, &msg);
if (unlikely(mpi_error != MPI_SUCCESS)) goto err_create_packed_msg;
MPID_PSP_packed_msg_pack(origin_addr, origin_count, origin_datatype, &msg);
target_buf = (char *) ri->base_addr + ri->disp_unit * target_disp;
if (0 && MPID_PSP_Datatype_is_contig(&dt_info)) { /* ToDo: reenable pscom buildin rma_write */
/* Contig message. Use pscom buildin rma */
pscom_request_t *req = pscom_request_create(0, 0);
req->data_len = msg.msg_sz;
req->data = msg.msg;
req->connection = ri->con;
/* ToDo: need a new io_done. inside io_done, call MPID_PSP_packed_msg_cleanup(msg)!!! */
req->ops.io_done = pscom_request_free;
req->xheader.rma_write.dest = target_buf;
pscom_post_rma_write(req);
/* win_ptr->rma_puts_accs[target_rank]++; / ToDo: Howto receive this? */
} else {
unsigned int encode_dt_size = MPID_PSP_Datatype_get_size(&dt_info);
unsigned int xheader_len = sizeof(MPID_PSCOM_XHeader_Rma_put_t) + encode_dt_size;
pscom_request_t *req = pscom_request_create(xheader_len, sizeof(pscom_request_put_send_t));
MPID_PSCOM_XHeader_Rma_put_t *xheader = &req->xheader.user.put;
/* encoded datatype too large for xheader? */
assert(xheader_len < (1<<(8*sizeof(((struct PSCOM_header_net*)0)->xheader_len))));
req->user->type.put_send.msg = msg;
req->user->type.put_send.win_ptr = win_ptr;
MPID_PSP_Datatype_encode(&dt_info, &xheader->encoded_type);
xheader->common.tag = 0;
xheader->common.context_id = 0;
xheader->common.type = MPID_PSP_MSGTYPE_RMA_PUT;
xheader->common._reserved_ = 0;
xheader->common.src_rank = win_ptr->rank;
/* xheader->target_disp = target_disp; */
xheader->target_count = target_count;
xheader->target_buf = target_buf;
/* xheader->epoch = ri->epoch_origin; */
xheader->win_ptr = ri->win_ptr; /* remote win_ptr */
req->xheader_len = xheader_len;
req->data = msg.msg;
req->data_len = msg.msg_sz;
req->ops.io_done = rma_put_done;
req->user->type.put_send.target_rank = target_rank;
req->connection = ri->con;
win_ptr->rma_local_pending_cnt++;
win_ptr->rma_local_pending_rank[target_rank]++;
win_ptr->rma_puts_accs[target_rank]++;
if(request) {
MPIR_Request *mpid_req = *request;
/* TODO: Use a new and 'put_send'-dedicated MPID_DEV_Request_create() */
/* instead of allocating and overloading a common send request. */
pscom_request_free(mpid_req->dev.kind.common.pscom_req);
mpid_req->dev.kind.common.pscom_req = req;
MPIR_Request_add_ref(mpid_req);
req->user->type.put_send.mpid_req = mpid_req;
} else {
req->user->type.put_send.mpid_req = NULL;
}
pscom_post_send(req);
}
fn_exit:
return MPI_SUCCESS;
fn_completed:
if(request) {
MPIDI_PSP_Request_set_completed(*request);
}
return MPI_SUCCESS;
/* --- */
err_exit:
if(request) {
MPIDI_PSP_Request_set_completed(*request);
MPIR_Request_free(*request);
}
return mpi_error;
/* --- */
err_create_packed_msg:
goto err_exit;
err_local_copy:
goto err_exit;
err_sync_rma:
goto err_exit;
}
/*@
MPI_Request_free - Frees a communication request object
Input Parameters:
. request - communication request (handle)
Notes:
This routine is normally used to free inactive persistent requests created with
either 'MPI_Recv_init' or 'MPI_Send_init' and friends. It `is` also
permissible to free an active request. However, once freed, the request can no
longer be used in a wait or test routine (e.g., 'MPI_Wait') to determine
completion.
This routine may also be used to free a non-persistent requests such as those
created with 'MPI_Irecv' or 'MPI_Isend' and friends. Like active persistent
requests, once freed, the request can no longer be used with test/wait routines
to determine completion.
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_REQUEST
.N MPI_ERR_ARG
.see also: MPI_Isend, MPI_Irecv, MPI_Issend, MPI_Ibsend, MPI_Irsend,
MPI_Recv_init, MPI_Send_init, MPI_Ssend_init, MPI_Rsend_init, MPI_Wait,
MPI_Test, MPI_Waitall, MPI_Waitany, MPI_Waitsome, MPI_Testall, MPI_Testany,
MPI_Testsome
@*/
int MPI_Request_free(MPI_Request * request)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *request_ptr = NULL;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_REQUEST_FREE);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_TERSE_ENTER(MPID_STATE_MPI_REQUEST_FREE);
/* Validate handle parameters needing to be converted */
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_ARGNULL(request, "request", mpi_errno);
MPIR_ERRTEST_REQUEST(*request, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPIR_Request_get_ptr(*request, request_ptr);
/* Validate object pointers if error checking is enabled */
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate request_ptr */
MPIR_Request_valid_ptr(request_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
MPID_Progress_poke();
switch (request_ptr->kind) {
case MPIR_REQUEST_KIND__SEND:
{
MPII_SENDQ_FORGET(request_ptr);
break;
}
case MPIR_REQUEST_KIND__RECV:
{
break;
}
case MPIR_REQUEST_KIND__PREQUEST_SEND:
{
/* If this is an active persistent request, we must also
* release the partner request. */
if (request_ptr->u.persist.real_request != NULL) {
if (request_ptr->u.persist.real_request->kind == MPIR_REQUEST_KIND__GREQUEST) {
/* This is needed for persistent Bsend requests */
mpi_errno = MPIR_Grequest_free(request_ptr->u.persist.real_request);
}
MPIR_Request_free(request_ptr->u.persist.real_request);
}
break;
}
case MPIR_REQUEST_KIND__PREQUEST_RECV:
{
/* If this is an active persistent request, we must also
* release the partner request. */
if (request_ptr->u.persist.real_request != NULL) {
MPIR_Request_free(request_ptr->u.persist.real_request);
}
break;
}
case MPIR_REQUEST_KIND__GREQUEST:
{
mpi_errno = MPIR_Grequest_free(request_ptr);
break;
}
/* --BEGIN ERROR HANDLING-- */
default:
{
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
FCNAME, __LINE__, MPI_ERR_OTHER,
"**request_invalid_kind",
"**request_invalid_kind %d", request_ptr->kind);
break;
}
/* --END ERROR HANDLING-- */
}
MPIR_Request_free(request_ptr);
*request = MPI_REQUEST_NULL;
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPIR_FUNC_TERSE_EXIT(MPID_STATE_MPI_REQUEST_FREE);
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
#ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE,
FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_request_free",
"**mpi_request_free %p", request);
}
#endif
mpi_errno = MPIR_Err_return_comm(0, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Sendrecv_replace - Sends and receives using a single buffer
Input Parameters:
+ count - number of elements in send and receive buffer (integer)
. datatype - type of elements in send and receive buffer (handle)
. dest - rank of destination (integer)
. sendtag - send message tag (integer)
. source - rank of source (integer)
. recvtag - receive message tag (integer)
- comm - communicator (handle)
Output Parameters:
+ buf - initial address of send and receive buffer (choice)
- status - status object (Status)
.N ThreadSafe
.N Fortran
.N FortranStatus
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_COUNT
.N MPI_ERR_TYPE
.N MPI_ERR_TAG
.N MPI_ERR_RANK
.N MPI_ERR_TRUNCATE
.N MPI_ERR_EXHAUSTED
@*/
int MPI_Sendrecv_replace(void *buf, int count, MPI_Datatype datatype,
int dest, int sendtag, int source, int recvtag,
MPI_Comm comm, MPI_Status *status)
{
static const char FCNAME[] = "MPI_Sendrecv_replace";
int mpi_errno = MPI_SUCCESS;
MPIR_Comm *comm_ptr = NULL;
MPIR_CHKLMEM_DECL(1);
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_SENDRECV_REPLACE);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_TERSE_PT2PT_ENTER_BOTH(MPID_STATE_MPI_SENDRECV_REPLACE);
/* Convert handles to MPI objects. */
MPIR_Comm_get_ptr(comm, comm_ptr);
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate communicator */
MPIR_Comm_valid_ptr( comm_ptr, mpi_errno, FALSE );
if (mpi_errno) goto fn_fail;
/* Validate count */
MPIR_ERRTEST_COUNT(count, mpi_errno);
/* Validate status (status_ignore is not the same as null) */
MPIR_ERRTEST_ARGNULL(status, "status", mpi_errno);
/* Validate tags */
MPIR_ERRTEST_SEND_TAG(sendtag, mpi_errno);
MPIR_ERRTEST_RECV_TAG(recvtag, mpi_errno);
/* Validate source and destination */
MPIR_ERRTEST_SEND_RANK(comm_ptr, dest, mpi_errno);
MPIR_ERRTEST_RECV_RANK(comm_ptr, source, mpi_errno);
/* Validate datatype handle */
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
/* Validate datatype object */
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN)
{
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(datatype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
}
/* Validate buffer */
MPIR_ERRTEST_USERBUFFER(buf,count,datatype,mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
# if defined(MPID_Sendrecv_replace)
{
mpi_errno = MPID_Sendrecv_replace(buf, count, datatype, dest,
sendtag, source, recvtag, comm_ptr,
status)
}
# else
{
MPIR_Request * sreq;
MPIR_Request * rreq;
void * tmpbuf = NULL;
MPI_Aint tmpbuf_size = 0;
MPI_Aint tmpbuf_count = 0;
if (count > 0 && dest != MPI_PROC_NULL)
{
MPIR_Pack_size_impl(count, datatype, &tmpbuf_size);
MPIR_CHKLMEM_MALLOC_ORJUMP(tmpbuf, void *, tmpbuf_size, mpi_errno, "temporary send buffer", MPL_MEM_BUFFER);
mpi_errno = MPIR_Pack_impl(buf, count, datatype, tmpbuf, tmpbuf_size, &tmpbuf_count);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
mpi_errno = MPID_Irecv(buf, count, datatype, source, recvtag,
comm_ptr, MPIR_CONTEXT_INTRA_PT2PT, &rreq);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
mpi_errno = MPID_Isend(tmpbuf, tmpbuf_count, MPI_PACKED, dest,
sendtag, comm_ptr, MPIR_CONTEXT_INTRA_PT2PT,
&sreq);
if (mpi_errno != MPI_SUCCESS)
{
/* --BEGIN ERROR HANDLING-- */
/* FIXME: should we cancel the pending (possibly completed) receive request or wait for it to complete? */
MPIR_Request_free(rreq);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
if (!MPIR_Request_is_complete(sreq) || !MPIR_Request_is_complete(rreq))
{
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (!MPIR_Request_is_complete(sreq) || !MPIR_Request_is_complete(rreq))
{
mpi_errno = MPID_Progress_wait(&progress_state);
if (mpi_errno != MPI_SUCCESS)
{
/* --BEGIN ERROR HANDLING-- */
MPID_Progress_end(&progress_state);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
}
MPID_Progress_end(&progress_state);
}
if (status != MPI_STATUS_IGNORE)
{
*status = rreq->status;
}
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = rreq->status.MPI_ERROR;
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = sreq->status.MPI_ERROR;
}
}
MPIR_Request_free(sreq);
MPIR_Request_free(rreq);
}
/*@
MPI_Ssend - Blocking synchronous send
Input Parameters:
+ buf - initial address of send buffer (choice)
. count - number of elements in send buffer (nonnegative integer)
. datatype - datatype of each send buffer element (handle)
. dest - rank of destination (integer)
. tag - message tag (integer)
- comm - communicator (handle)
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_COUNT
.N MPI_ERR_TYPE
.N MPI_ERR_TAG
.N MPI_ERR_RANK
@*/
int MPI_Ssend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm)
{
static const char FCNAME[] = "MPI_Ssend";
int mpi_errno = MPI_SUCCESS;
MPIR_Comm *comm_ptr = NULL;
MPIR_Request * request_ptr = NULL;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_SSEND);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_TERSE_PT2PT_ENTER_FRONT(MPID_STATE_MPI_SSEND);
/* Validate handle parameters needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPIR_Comm_get_ptr( comm, comm_ptr );
/* Validate parameters if error checking is enabled */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_Comm_valid_ptr( comm_ptr, mpi_errno, FALSE );
if (mpi_errno) goto fn_fail;
MPIR_ERRTEST_COUNT(count, mpi_errno);
MPIR_ERRTEST_SEND_RANK(comm_ptr, dest, mpi_errno);
MPIR_ERRTEST_SEND_TAG(tag, mpi_errno);
/* Validate datatype handle */
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
/* Validate datatype object */
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN)
{
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(datatype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
}
/* Validate buffer */
MPIR_ERRTEST_USERBUFFER(buf,count,datatype,mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
MPIR_CONTEXT_INTRA_PT2PT, &request_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
if (request_ptr == NULL)
{
goto fn_exit;
}
/* If a request was returned, then we need to block until the request
is complete */
mpi_errno = MPIR_Progress_wait_request(request_ptr);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = request_ptr->status.MPI_ERROR;
MPIR_Request_free(request_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPIR_FUNC_TERSE_PT2PT_EXIT(MPID_STATE_MPI_SSEND);
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_ssend",
"**mpi_ssend %p %d %D %i %t %C", buf, count, datatype, dest, tag, comm);
}
# endif
mpi_errno = MPIR_Err_return_comm( comm_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDI_CH3_PktHandler_Close( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
intptr_t *buflen, MPIR_Request **rreqp )
{
MPIDI_CH3_Pkt_close_t * close_pkt = &pkt->close;
int mpi_errno = MPI_SUCCESS;
if (vc->state == MPIDI_VC_STATE_LOCAL_CLOSE)
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_close_t * resp_pkt = &upkt.close;
MPIR_Request * resp_sreq;
MPIDI_Pkt_init(resp_pkt, MPIDI_CH3_PKT_CLOSE);
resp_pkt->ack = TRUE;
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL,"sending close(TRUE) to %d",
vc->pg_rank);
mpi_errno = MPIDI_CH3_iStartMsg(vc, resp_pkt, sizeof(*resp_pkt), &resp_sreq);
MPIR_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|send_close_ack");
if (resp_sreq != NULL)
{
/* There is still another reference being held by the channel. It
will not be released until the pkt is actually sent. */
MPIR_Request_free(resp_sreq);
}
}
if (close_pkt->ack == FALSE)
{
if (vc->state == MPIDI_VC_STATE_LOCAL_CLOSE)
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL,
"received close(FALSE) from %d, moving to CLOSE_ACKED.",
vc->pg_rank);
MPIDI_CHANGE_VC_STATE(vc, CLOSE_ACKED);
}
else /* (vc->state == MPIDI_VC_STATE_ACTIVE) */
{
if (vc->state != MPIDI_VC_STATE_ACTIVE)
{
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_DISCONNECT, TYPICAL, (MPL_DBG_FDEST, "Unexpected state %s in vc %p (rank=%d) (expecting MPIDI_VC_STATE_ACTIVE)\n", MPIDI_VC_GetStateString(vc->state), vc, vc->pg_rank ));
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL,
"received close(FALSE) from %d, moving to REMOTE_CLOSE.",
vc->pg_rank);
}
MPIR_Assert(vc->state == MPIDI_VC_STATE_ACTIVE);
MPIDI_CHANGE_VC_STATE(vc, REMOTE_CLOSE);
}
}
else /* (close_pkt->ack == TRUE) */
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_DISCONNECT,TYPICAL,
"received close(TRUE) from %d, moving to CLOSED.",
vc->pg_rank);
MPIR_Assert (vc->state == MPIDI_VC_STATE_LOCAL_CLOSE ||
vc->state == MPIDI_VC_STATE_CLOSE_ACKED);
MPIDI_CHANGE_VC_STATE(vc, CLOSED);
/* For example, with sockets, Connection_terminate will close
the socket */
mpi_errno = MPIDI_CH3_Connection_terminate(vc);
}
*buflen = sizeof(MPIDI_CH3_Pkt_t);
*rreqp = NULL;
fn_fail:
return mpi_errno;
}
int MPIDI_CH3_PktHandler_RndvClrToSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
intptr_t *buflen, MPIR_Request **rreqp )
{
MPIDI_CH3_Pkt_rndv_clr_to_send_t * cts_pkt = &pkt->rndv_clr_to_send;
MPIR_Request * sreq;
MPIR_Request * rts_sreq;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_rndv_send_t * rs_pkt = &upkt.rndv_send;
int dt_contig;
MPI_Aint dt_true_lb;
intptr_t data_sz;
MPIDU_Datatype* dt_ptr;
int mpi_errno = MPI_SUCCESS;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"received rndv CTS pkt");
MPIR_Request_get_ptr(cts_pkt->sender_req_id, sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,TERSE,(MPL_DBG_FDEST,"received cts, count=" MPI_AINT_FMT_DEC_SPEC "\n", sreq->dev.user_count));
sreq->dev.OnDataAvail = 0;
sreq->dev.OnFinal = 0;
/* Release the RTS request if one exists.
MPID_Request_fetch_and_clear_rts_sreq() needs to be atomic to
prevent
cancel send from cancelling the wrong (future) request.
If MPID_Request_fetch_and_clear_rts_sreq() returns a NULL
rts_sreq, then MPID_Cancel_send() is responsible for releasing
the RTS request object. */
MPIDI_Request_fetch_and_clear_rts_sreq(sreq, &rts_sreq);
if (rts_sreq != NULL)
{
MPIR_Request_free(rts_sreq);
}
*buflen = sizeof(MPIDI_CH3_Pkt_t);
MPIDI_Pkt_init(rs_pkt, MPIDI_CH3_PKT_RNDV_SEND);
rs_pkt->receiver_req_id = cts_pkt->receiver_req_id;
MPIDI_Datatype_get_info(sreq->dev.user_count, sreq->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
if (dt_contig)
{
MPL_IOV iov[MPL_IOV_LIMIT];
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"sending contiguous rndv data, data_sz=%" PRIdPTR,
data_sz));
iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)rs_pkt;
iov[0].MPL_IOV_LEN = sizeof(*rs_pkt);
iov[1].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)((char *)sreq->dev.user_buf + dt_true_lb);
iov[1].MPL_IOV_LEN = data_sz;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iSendv(vc, sreq, iov, 2);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
MPIR_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|senddata");
}
else
{
sreq->dev.segment_ptr = MPIDU_Segment_alloc( );
MPIR_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDU_Segment_alloc");
MPIDU_Segment_init(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = vc->sendNoncontig_fn(vc, sreq, rs_pkt, sizeof(*rs_pkt));
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
MPIR_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|senddata");
}
*rreqp = NULL;
fn_fail:
return mpi_errno;
}
/*@
MPI_Sendrecv - Sends and receives a message
Input Parameters:
+ sendbuf - initial address of send buffer (choice)
. sendcount - number of elements in send buffer (integer)
. sendtype - type of elements in send buffer (handle)
. dest - rank of destination (integer)
. sendtag - send tag (integer)
. recvcount - number of elements in receive buffer (integer)
. recvtype - type of elements in receive buffer (handle)
. source - rank of source (integer)
. recvtag - receive tag (integer)
- comm - communicator (handle)
Output Parameters:
+ recvbuf - initial address of receive buffer (choice)
- status - status object (Status). This refers to the receive operation.
.N ThreadSafe
.N Fortran
.N FortranStatus
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_COUNT
.N MPI_ERR_TYPE
.N MPI_ERR_TAG
.N MPI_ERR_RANK
@*/
int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
int source, int recvtag, MPI_Comm comm, MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Comm *comm_ptr = NULL;
MPIR_Request *sreq = NULL;
MPIR_Request *rreq = NULL;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_SENDRECV);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(VCI_GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_TERSE_PT2PT_ENTER_BOTH(MPID_STATE_MPI_SENDRECV);
/* Validate handle parameters needing to be converted */
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* Convert handles to MPI objects. */
MPIR_Comm_get_ptr(comm, comm_ptr);
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate communicator */
MPIR_Comm_valid_ptr(comm_ptr, mpi_errno, FALSE);
if (mpi_errno)
goto fn_fail;
/* Validate count */
MPIR_ERRTEST_COUNT(sendcount, mpi_errno);
MPIR_ERRTEST_COUNT(recvcount, mpi_errno);
/* Validate status (status_ignore is not the same as null) */
MPIR_ERRTEST_ARGNULL(status, "status", mpi_errno);
/* Validate tags */
MPIR_ERRTEST_SEND_TAG(sendtag, mpi_errno);
MPIR_ERRTEST_RECV_TAG(recvtag, mpi_errno);
/* Validate source and destination */
if (comm_ptr) {
MPIR_ERRTEST_SEND_RANK(comm_ptr, dest, mpi_errno);
MPIR_ERRTEST_RECV_RANK(comm_ptr, source, mpi_errno);
}
/* Validate datatype handles */
MPIR_ERRTEST_DATATYPE(sendtype, "datatype", mpi_errno);
MPIR_ERRTEST_DATATYPE(recvtype, "datatype", mpi_errno);
/* Validate datatype objects */
if (HANDLE_GET_KIND(sendtype) != HANDLE_KIND_BUILTIN) {
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(sendtype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
}
if (HANDLE_GET_KIND(recvtype) != HANDLE_KIND_BUILTIN) {
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(recvtype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
}
/* Validate buffers */
MPIR_ERRTEST_USERBUFFER(sendbuf, sendcount, sendtype, mpi_errno);
MPIR_ERRTEST_USERBUFFER(recvbuf, recvcount, recvtype, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno =
MPID_Irecv(recvbuf, recvcount, recvtype, source, recvtag, comm_ptr,
MPIR_CONTEXT_INTRA_PT2PT, &rreq);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* FIXME - Performance for small messages might be better if MPID_Send() were used here instead of MPID_Isend() */
mpi_errno =
MPID_Isend(sendbuf, sendcount, sendtype, dest, sendtag, comm_ptr, MPIR_CONTEXT_INTRA_PT2PT,
&sreq);
if (mpi_errno != MPI_SUCCESS) {
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno == MPIX_ERR_NOREQ)
MPIR_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**nomem");
/* FIXME: should we cancel the pending (possibly completed) receive request or wait for it to complete? */
MPIR_Request_free(rreq);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
if (!MPIR_Request_is_complete(sreq) || !MPIR_Request_is_complete(rreq)) {
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (!MPIR_Request_is_complete(sreq) || !MPIR_Request_is_complete(rreq)) {
mpi_errno = MPID_Progress_wait(&progress_state);
if (mpi_errno != MPI_SUCCESS) {
/* --BEGIN ERROR HANDLING-- */
MPID_Progress_end(&progress_state);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
if (unlikely(MPIR_Request_is_anysrc_mismatched(rreq))) {
/* --BEGIN ERROR HANDLING-- */
mpi_errno = MPIR_Request_handle_proc_failed(rreq);
if (!MPIR_Request_is_complete(sreq)) {
MPID_Cancel_send(sreq);
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
}
goto fn_fail;
/* --END ERROR HANDLING-- */
}
}
MPID_Progress_end(&progress_state);
}
mpi_errno = rreq->status.MPI_ERROR;
MPIR_Request_extract_status(rreq, status);
MPIR_Request_free(rreq);
if (mpi_errno == MPI_SUCCESS) {
mpi_errno = sreq->status.MPI_ERROR;
}
MPIR_Request_free(sreq);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPIR_FUNC_TERSE_PT2PT_EXIT_BOTH(MPID_STATE_MPI_SENDRECV);
MPID_THREAD_CS_EXIT(VCI_GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
#ifdef HAVE_ERROR_CHECKING
{
mpi_errno =
MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, __func__, __LINE__, MPI_ERR_OTHER,
"**mpi_sendrecv",
"**mpi_sendrecv %p %d %D %i %t %p %d %D %i %t %C %p", sendbuf,
sendcount, sendtype, dest, sendtag, recvbuf, recvcount, recvtype,
source, recvtag, comm, status);
}
#endif
mpi_errno = MPIR_Err_return_comm(comm_ptr, __func__, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Recv - Blocking receive for a message
Output Parameters:
+ buf - initial address of receive buffer (choice)
- status - status object (Status)
Input Parameters:
+ count - maximum number of elements in receive buffer (integer)
. datatype - datatype of each receive buffer element (handle)
. source - rank of source (integer)
. tag - message tag (integer)
- comm - communicator (handle)
Notes:
The 'count' argument indicates the maximum length of a message; the actual
length of the message can be determined with 'MPI_Get_count'.
.N ThreadSafe
.N Fortran
.N FortranStatus
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_TYPE
.N MPI_ERR_COUNT
.N MPI_ERR_TAG
.N MPI_ERR_RANK
@*/
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Comm *comm_ptr = NULL;
MPIR_Request *request_ptr = NULL;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_RECV);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(VNI_GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_TERSE_PT2PT_ENTER_BACK(MPID_STATE_MPI_RECV);
/* Validate handle parameters needing to be converted */
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
/* NOTE: MPI_STATUS_IGNORE != NULL */
MPIR_ERRTEST_ARGNULL(status, "status", mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPIR_Comm_get_ptr(comm, comm_ptr);
/* Validate parameters if error checking is enabled */
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_Comm_valid_ptr(comm_ptr, mpi_errno, FALSE);
if (mpi_errno)
goto fn_fail;
MPIR_ERRTEST_COUNT(count, mpi_errno);
MPIR_ERRTEST_RECV_RANK(comm_ptr, source, mpi_errno);
MPIR_ERRTEST_RECV_TAG(tag, mpi_errno);
/* Validate datatype handle */
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
/* Validate datatype object */
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(datatype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
}
/* Validate buffer */
MPIR_ERRTEST_USERBUFFER(buf, count, datatype, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
/* MT: Note that MPID_Recv may release the SINGLE_CS if it
* decides to block internally. MPID_Recv in that case will
* re-aquire the SINGLE_CS before returnning */
mpi_errno = MPID_Recv(buf, count, datatype, source, tag, comm_ptr,
MPIR_CONTEXT_INTRA_PT2PT, status, &request_ptr);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
if (request_ptr == NULL) {
goto fn_exit;
}
mpi_errno = MPID_Wait(request_ptr, MPI_STATUS_IGNORE);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
mpi_errno = request_ptr->status.MPI_ERROR;
MPIR_Request_extract_status(request_ptr, status);
MPIR_Request_free(request_ptr);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPIR_FUNC_TERSE_PT2PT_EXIT_BACK(MPID_STATE_MPI_RECV);
MPID_THREAD_CS_EXIT(VNI_GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
#ifdef HAVE_ERROR_CHECKING
{
mpi_errno =
MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_recv", "**mpi_recv %p %d %D %i %t %C %p", buf, count,
datatype, source, tag, comm, status);
}
#endif
mpi_errno = MPIR_Err_return_comm(comm_ptr, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/* MPIDI_CH3_EagerSyncNoncontigSend - Eagerly send noncontiguous data in
synchronous mode.
Some implementations may choose to use Rendezvous sends (see ch3u_rndv.c)
for all Synchronous sends (MPI_Issend and MPI_Ssend). An eager
synchronous send eliminates one of the handshake messages, but
most application codes should not be using synchronous sends in
performance-critical operations.
*/
int MPIDI_CH3_EagerSyncNoncontigSend( MPIR_Request **sreq_p,
const void * buf, int count,
MPI_Datatype datatype, intptr_t data_sz,
int dt_contig, MPI_Aint dt_true_lb,
int rank,
int tag, MPIR_Comm * comm,
int context_offset )
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_sync_send_t * const es_pkt = &upkt.eager_sync_send;
MPIDI_VC_t * vc;
MPIR_Request *sreq = *sreq_p;
/* MT FIXME what are the two operations we are waiting for? the send and
* the sync response? */
MPIR_cc_set(&sreq->cc, 2);
sreq->dev.OnDataAvail = 0;
sreq->dev.OnFinal = 0;
MPIDI_Pkt_init(es_pkt, MPIDI_CH3_PKT_EAGER_SYNC_SEND);
es_pkt->match.parts.rank = comm->rank;
es_pkt->match.parts.tag = tag;
es_pkt->match.parts.context_id = comm->context_id + context_offset;
es_pkt->sender_req_id = sreq->handle;
es_pkt->data_sz = data_sz;
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(es_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPL_DBG_MSGPKT(vc,tag,es_pkt->match.parts.context_id,rank,data_sz,"EagerSync");
if (dt_contig)
{
MPL_IOV iov[2];
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"sending contiguous sync eager message, data_sz=%" PRIdPTR,
data_sz));
iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)es_pkt;
iov[0].MPL_IOV_LEN = sizeof(*es_pkt);
iov[1].MPL_IOV_BUF = (MPL_IOV_BUF_CAST) ((char *)buf + dt_true_lb);
iov[1].MPL_IOV_LEN = data_sz;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iSendv(vc, sreq, iov, 2);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
/* Make sure to destroy the request before setting the pointer to
* NULL, otherwise we lose the handle on the request */
MPIR_Request_free(sreq);
*sreq_p = NULL;
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|eagermsg");
}
/* --END ERROR HANDLING-- */
}
else
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_OTHER,VERBOSE,
"sending non-contiguous sync eager message, data_sz=%" PRIdPTR,
data_sz);
sreq->dev.segment_ptr = MPIDU_Segment_alloc( );
MPIR_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPIDU_Segment_alloc");
MPIDU_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = vc->sendNoncontig_fn(vc, sreq, es_pkt, sizeof(MPIDI_CH3_Pkt_eager_sync_send_t));
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
fn_exit:
return mpi_errno;
fn_fail:
*sreq_p = NULL;
goto fn_exit;
}
int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
intptr_t *buflen, MPIR_Request **rreqp )
{
MPIDI_CH3_Pkt_eager_send_t * es_pkt = &pkt->eager_send;
MPIR_Request * rreq;
int found;
int complete;
char *data_buf;
intptr_t data_len;
int mpi_errno = MPI_SUCCESS;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"received eager sync send pkt, sreq=0x%08x, rank=%d, tag=%d, context=%d",
es_pkt->sender_req_id, es_pkt->match.parts.rank,
es_pkt->match.parts.tag,
es_pkt->match.parts.context_id));
MPL_DBG_MSGPKT(vc,es_pkt->match.parts.tag,es_pkt->match.parts.context_id,
es_pkt->match.parts.rank,es_pkt->data_sz,
"ReceivedEagerSync");
rreq = MPIDI_CH3U_Recvq_FDP_or_AEU(&es_pkt->match, &found);
MPIR_ERR_CHKANDJUMP1(!rreq, mpi_errno,MPI_ERR_OTHER, "**nomemreq", "**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
/* If the completion counter is 0, that means that the communicator to
* which this message is being sent has been revoked and we shouldn't
* bother finishing this. */
if (!found && MPIR_cc_get(rreq->cc) == 0) {
*rreqp = NULL;
goto fn_fail;
}
set_request_info(rreq, es_pkt, MPIDI_REQUEST_EAGER_MSG);
data_len = ((*buflen - sizeof(MPIDI_CH3_Pkt_t) >= rreq->dev.recv_data_sz)
? rreq->dev.recv_data_sz : *buflen - sizeof(MPIDI_CH3_Pkt_t));
data_buf = (char *)pkt + sizeof(MPIDI_CH3_Pkt_t);
if (found)
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_sync_ack_t * const esa_pkt = &upkt.eager_sync_ack;
MPIR_Request * esa_req;
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_found( rreq, data_buf,
&data_len, &complete );
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_EAGER_SYNC_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else
{
*rreqp = rreq;
}
}
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"sending eager sync ack");
MPIDI_Pkt_init(esa_pkt, MPIDI_CH3_PKT_EAGER_SYNC_ACK);
esa_pkt->sender_req_id = rreq->dev.sender_req_id;
/* Because this is a packet handler, it is already within a CH3 CS */
/* MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex); */
mpi_errno = MPIDI_CH3_iStartMsg(vc, esa_pkt, sizeof(*esa_pkt), &esa_req);
/* MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex); */
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**ch3|syncack");
}
if (esa_req != NULL) {
MPIR_Request_free(esa_req);
}
}
else
{
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_unexpected( rreq, data_buf,
&data_len, &complete );
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_EAGER_SYNC_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else
{
*rreqp = rreq;
}
}
MPIDI_Request_set_sync_send_flag(rreq, TRUE);
}
fn_fail:
return mpi_errno;
}
/*
* This function does all of the work or either revoking the communciator for
* the first time or keeping track of an ongoing revocation.
*
* comm_ptr - The communicator being revoked
* is_remote - If we received the revocation from a remote process, this should
* be set to true. This way we'll know to decrement the counter twice
* (once for our local revocation and once for the remote).
*/
int MPID_Comm_revoke(MPIR_Comm *comm_ptr, int is_remote)
{
MPIDI_VC_t *vc;
MPL_IOV iov[MPL_IOV_LIMIT];
int mpi_errno = MPI_SUCCESS;
int i, size, my_rank;
MPIR_Request *request;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_revoke_t *revoke_pkt = &upkt.revoke;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_COMM_REVOKE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_COMM_REVOKE);
if (0 == comm_ptr->revoked) {
/* Mark the communicator as revoked locally */
comm_ptr->revoked = 1;
if (comm_ptr->node_comm) comm_ptr->node_comm->revoked = 1;
if (comm_ptr->node_roots_comm) comm_ptr->node_roots_comm->revoked = 1;
/* Start a counter to track how many revoke messages we've received from
* other ranks */
comm_ptr->dev.waiting_for_revoke = comm_ptr->local_size - 1 - is_remote; /* Subtract the processes who already know about the revoke */
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER, VERBOSE, (MPL_DBG_FDEST, "Comm %08x waiting_for_revoke: %d", comm_ptr->handle, comm_ptr->dev.waiting_for_revoke));
/* Keep a reference to this comm so it doesn't get destroyed while
* it's being revoked */
MPIR_Comm_add_ref(comm_ptr);
/* Send out the revoke message */
MPIDI_Pkt_init(revoke_pkt, MPIDI_CH3_PKT_REVOKE);
revoke_pkt->revoked_comm = comm_ptr->context_id;
size = comm_ptr->remote_size;
my_rank = comm_ptr->rank;
for (i = 0; i < size; i++) {
if (i == my_rank) continue;
request = NULL;
MPIDI_Comm_get_vc_set_active(comm_ptr, i, &vc);
iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST) revoke_pkt;
iov[0].MPL_IOV_LEN = sizeof(*revoke_pkt);
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsgv(vc, iov, 1, &request);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno) comm_ptr->dev.waiting_for_revoke--;
if (NULL != request)
/* We don't need to keep a reference to this request. The
* progress engine will keep a reference until it completes
* later */
MPIR_Request_free(request);
}
/* Check to see if we are done revoking */
if (comm_ptr->dev.waiting_for_revoke == 0) {
MPIR_Comm_release(comm_ptr);
}
/* Go clean up all of the existing operations involving this
* communicator. This includes completing existing MPI requests, MPID
* requests, and cleaning up the unexpected queue to make sure there
* aren't any unexpected messages hanging around. */
/* Clean up the receive and unexpected queues */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
MPIDI_CH3U_Clean_recvq(comm_ptr);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
} else if (is_remote) { /* If this is local, we've already revoked and don't need to do it again. */
/* Decrement the revoke counter */
comm_ptr->dev.waiting_for_revoke--;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER, VERBOSE, (MPL_DBG_FDEST, "Comm %08x waiting_for_revoke: %d", comm_ptr->handle, comm_ptr->dev.waiting_for_revoke));
/* Check to see if we are done revoking */
if (comm_ptr->dev.waiting_for_revoke == 0) {
MPIR_Comm_release(comm_ptr);
}
}
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_COMM_REVOKE);
return MPI_SUCCESS;
}
int MPIDI_CH3_PktHandler_RndvReqToSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
intptr_t *buflen, MPIR_Request **rreqp )
{
MPIR_Request * rreq;
int found;
MPIDI_CH3_Pkt_rndv_req_to_send_t * rts_pkt = &pkt->rndv_req_to_send;
int mpi_errno = MPI_SUCCESS;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"received rndv RTS pkt, sreq=0x%08x, rank=%d, tag=%d, context=%d, data_sz=%" PRIdPTR,
rts_pkt->sender_req_id, rts_pkt->match.parts.rank,
rts_pkt->match.parts.tag,
rts_pkt->match.parts.context_id, rts_pkt->data_sz));
MPL_DBG_MSGPKT(vc,rts_pkt->match.parts.tag,rts_pkt->match.parts.context_id,
rts_pkt->match.parts.rank,rts_pkt->data_sz,
"ReceivedRndv");
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
rreq = MPIDI_CH3U_Recvq_FDP_or_AEU(&rts_pkt->match, &found);
MPIR_ERR_CHKANDJUMP1(!rreq, mpi_errno,MPI_ERR_OTHER, "**nomemreq", "**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
/* If the completion counter is 0, that means that the communicator to
* which this message is being sent has been revoked and we shouldn't
* bother finishing this. */
if (!found && MPIR_cc_get(rreq->cc) == 0) {
*rreqp = NULL;
goto fn_fail;
}
set_request_info(rreq, rts_pkt, MPIDI_REQUEST_RNDV_MSG);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
*buflen = sizeof(MPIDI_CH3_Pkt_t);
if (found)
{
MPIR_Request * cts_req;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_rndv_clr_to_send_t * cts_pkt = &upkt.rndv_clr_to_send;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"posted request found");
/* FIXME: What if the receive user buffer is not big enough to
hold the data about to be cleared for sending? */
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"sending rndv CTS packet");
MPIDI_Pkt_init(cts_pkt, MPIDI_CH3_PKT_RNDV_CLR_TO_SEND);
cts_pkt->sender_req_id = rts_pkt->sender_req_id;
cts_pkt->receiver_req_id = rreq->handle;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, cts_pkt, sizeof(*cts_pkt), &cts_req);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**ch3|ctspkt");
}
if (cts_req != NULL) {
MPIR_Request_free(cts_req);
}
}
else
{
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"unexpected request allocated");
/*
* A MPID_Probe() may be waiting for the request we just
* inserted, so we need to tell the progress engine to exit.
*
* FIXME: This will cause MPID_Progress_wait() to return to the
* MPI layer each time an unexpected RTS packet is
* received. MPID_Probe() should atomically increment a
* counter and MPIDI_CH3_Progress_signal_completion()
* should only be called if that counter is greater than zero.
*/
MPIDI_CH3_Progress_signal_completion();
}
*rreqp = NULL;
fn_fail:
return mpi_errno;
}
