int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq;
MPIDI_nem_ptl_pkt_cancel_send_resp_t *resp_pkt = (MPIDI_nem_ptl_pkt_cancel_send_resp_t *)pkt;
int i, ret;
MPID_Request_get_ptr(resp_pkt->sender_req_id, sreq);
if (resp_pkt->ack) {
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* remove/free any remaining get MEs and handles */
for (i = 0; i < REQ_PTL(sreq)->num_gets; i++) {
ret = PtlMEUnlink(REQ_PTL(sreq)->get_me_p[i]);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"message cancelled");
} else {
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIO_Waitany(int count, MPIO_Request requests[], int *index,
MPI_Status *status)
{
int i, flag, err;
MPID_THREADPRIV_DECL;
ROMIO_THREAD_CS_ENTER();
if (count == 1) {
err = MPIO_Wait( requests, status );
if (!err) *index = 0;
goto fn_exit;
}
/* Check for no active requests */
for (i=0; i<count; i++) {
if (requests[i] != MPIO_REQUEST_NULL) {
break;
}
}
if (i == count) {
*index = MPI_UNDEFINED;
#ifdef MPICH
/* need to set empty status */
if (status != MPI_STATUS_IGNORE) {
status->MPI_SOURCE = MPI_ANY_SOURCE;
status->MPI_TAG = MPI_ANY_TAG;
MPIR_STATUS_SET_COUNT(*status, 0);
MPIR_STATUS_SET_CANCEL_BIT(*status, 0);
}
#endif
err = MPI_SUCCESS;
goto fn_exit;
}
err = MPI_SUCCESS;
do {
flag = 0;
for (i=0; i<count; i++) {
if (requests[i] != MPIO_REQUEST_NULL) {
err = MPIO_Test( &requests[i], &flag, status );
if (flag) {
if (!err) *index = i;
break;
}
}
}
} while (flag == 0);
fn_exit:
ROMIO_THREAD_CS_EXIT();
return err;
}
/*@
MPI_Status_set_cancelled - Sets the cancelled state associated with a
Status object
Input Parameters:
+ status - status to associate cancel flag with (Status)
- flag - if true indicates request was cancelled (logical)
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_ARG
@*/
int MPI_Status_set_cancelled(MPI_Status *status, int flag)
{
#ifdef HAVE_ERROR_CHECKING
static const char FCNAME[] = "MPI_Status_set_cancelled";
#endif
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_STATUS_SET_CANCELLED);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIR_FUNC_TERSE_ENTER(MPID_STATE_MPI_STATUS_SET_CANCELLED);
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_ARGNULL( status, "status", mpi_errno );
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
MPIR_STATUS_SET_CANCEL_BIT(*status, flag ? TRUE : FALSE);
/* ... end of body of routine ... */
#ifdef HAVE_ERROR_CHECKING
fn_exit:
#endif
MPIR_FUNC_TERSE_EXIT(MPID_STATE_MPI_STATUS_SET_CANCELLED);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
fn_fail:
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_status_set_cancelled",
"**mpi_status_set_cancelled %p %d", status, flag);
}
mpi_errno = MPIR_Err_return_comm( 0, FCNAME, mpi_errno );
goto fn_exit;
# endif
/* --END ERROR HANDLING-- */
}
int MPID_Cancel_recv(MPIR_Request * rreq)
{
int netmod_cancelled = TRUE;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_CANCEL_RECV);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_CANCEL_RECV);
MPIR_Assert(rreq->kind == MPIR_REQUEST_KIND__RECV);
/* If the netmod has its own cancel_recv function, we need to call
it here. ANYSOURCE cancels (netmod and otherwise) are handled by
MPIDI_CH3U_Recvq_DP below. */
#ifdef ENABLE_COMM_OVERRIDES
if (rreq->dev.match.parts.rank != MPI_ANY_SOURCE)
{
MPIDI_VC_t *vc;
MPIDI_Comm_get_vc_set_active(rreq->comm, rreq->dev.match.parts.rank, &vc);
if (vc->comm_ops && vc->comm_ops->cancel_recv)
netmod_cancelled = !vc->comm_ops->cancel_recv(NULL, rreq);
}
#endif
if (netmod_cancelled && MPIDI_CH3U_Recvq_DP(rreq))
{
MPL_DBG_MSG_P(MPIDI_CH3_DBG_OTHER,VERBOSE,
"request 0x%08x cancelled", rreq->handle);
MPIR_STATUS_SET_CANCEL_BIT(rreq->status, TRUE);
MPIR_STATUS_SET_COUNT(rreq->status, 0);
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else
{
MPL_DBG_MSG_P(MPIDI_CH3_DBG_OTHER,VERBOSE,
"request 0x%08x already matched, unable to cancel", rreq->handle);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_CANCEL_RECV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIO_Testall(int count, MPIO_Request requests[], int *flag,
MPI_Status statuses[])
{
int done, i, err;
ROMIO_THREAD_CS_ENTER();
if (count == 1) {
err = MPIO_Test( requests, flag, statuses );
goto fn_exit;
}
/* This is actually very difficult to do. We can't use MPIO_Test,
since we must change the requests only if *ALL* requests are complete
*/
/* FIXME: THIS IS NOT CORRECT (see above). But most applications won't
care */
done = 1;
for (i=0; i<count; i++) {
if (requests[i] != MPIO_REQUEST_NULL) {
err = MPIO_Test( &requests[i], flag, &statuses[i] );
if (!*flag) done = 0;
if (err) goto fn_exit;
}
else {
#ifdef MPICH
/* need to set empty status */
if (statuses != MPI_STATUSES_IGNORE) {
statuses[i].MPI_SOURCE = MPI_ANY_SOURCE;
statuses[i].MPI_TAG = MPI_ANY_TAG;
MPIR_STATUS_SET_COUNT(statuses[i], 0);
MPIR_STATUS_SET_CANCEL_BIT(statuses[i], 0);
}
#else
;
#endif
}
}
*flag = done;
err = MPI_SUCCESS;
fn_exit:
ROMIO_THREAD_CS_EXIT();
return err;
}
/*@
MPI_Recv - Blocking receive for a message
Output Parameters:
+ buf - initial address of receive buffer (choice)
- status - status object (Status)
Input Parameters:
+ count - maximum number of elements in receive buffer (integer)
. datatype - datatype of each receive buffer element (handle)
. source - rank of source (integer)
. tag - message tag (integer)
- comm - communicator (handle)
Notes:
The 'count' argument indicates the maximum length of a message; the actual
length of the message can be determined with 'MPI_Get_count'.
.N ThreadSafe
.N Fortran
.N FortranStatus
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_TYPE
.N MPI_ERR_COUNT
.N MPI_ERR_TAG
.N MPI_ERR_RANK
@*/
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
MPI_Comm comm, MPI_Status *status)
{
static const char FCNAME[] = "MPI_Recv";
int mpi_errno = MPI_SUCCESS;
MPID_Comm *comm_ptr = NULL;
MPID_Request * request_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_RECV);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_PT2PT_FUNC_ENTER_BACK(MPID_STATE_MPI_RECV);
/* Validate handle parameters needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
/* NOTE: MPI_STATUS_IGNORE != NULL */
MPIR_ERRTEST_ARGNULL(status, "status", mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPID_Comm_get_ptr( comm, comm_ptr );
/* Validate parameters if error checking is enabled */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPID_Comm_valid_ptr( comm_ptr, mpi_errno, FALSE );
if (mpi_errno) goto fn_fail;
MPIR_ERRTEST_COUNT(count, mpi_errno);
MPIR_ERRTEST_RECV_RANK(comm_ptr, source, mpi_errno);
MPIR_ERRTEST_RECV_TAG(tag, mpi_errno);
/* Validate datatype handle */
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
/* Validate datatype object */
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN)
{
MPID_Datatype *datatype_ptr = NULL;
MPID_Datatype_get_ptr(datatype, datatype_ptr);
MPID_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
MPID_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno) goto fn_fail;
}
/* Validate buffer */
MPIR_ERRTEST_USERBUFFER(buf,count,datatype,mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
/* MT: Note that MPID_Recv may release the SINGLE_CS if it
decides to block internally. MPID_Recv in that case will
re-aquire the SINGLE_CS before returnning */
mpi_errno = MPID_Recv(buf, count, datatype, source, tag, comm_ptr,
MPID_CONTEXT_INTRA_PT2PT, status, &request_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
if (request_ptr == NULL)
{
goto fn_exit;
}
/* If a request was returned, then we need to block until the request is
complete */
if (!MPID_Request_is_complete(request_ptr))
{
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (!MPID_Request_is_complete(request_ptr))
{
/* MT: Progress_wait may release the SINGLE_CS while it
waits */
mpi_errno = MPID_Progress_wait(&progress_state);
if (mpi_errno != MPI_SUCCESS)
{
/* --BEGIN ERROR HANDLING-- */
MPID_Progress_end(&progress_state);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
if (unlikely(MPIR_CVAR_ENABLE_FT &&
!MPID_Request_is_complete(request_ptr) &&
MPID_Request_is_anysource(request_ptr) &&
!MPID_Comm_AS_enabled(request_ptr->comm))) {
/* --BEGIN ERROR HANDLING-- */
MPID_Cancel_recv(request_ptr);
MPIR_STATUS_SET_CANCEL_BIT(request_ptr->status, FALSE);
MPIU_ERR_SET(request_ptr->status.MPI_ERROR, MPIX_ERR_PROC_FAILED, "**proc_failed");
mpi_errno = request_ptr->status.MPI_ERROR;
goto fn_fail;
/* --END ERROR HANDLING-- */
}
}
MPID_Progress_end(&progress_state);
}
mpi_errno = request_ptr->status.MPI_ERROR;
MPIR_Request_extract_status(request_ptr, status);
MPID_Request_release(request_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_PT2PT_FUNC_EXIT_BACK(MPID_STATE_MPI_RECV);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_recv",
"**mpi_recv %p %d %D %i %t %C %p", buf, count, datatype, source, tag, comm, status);
}
# endif
mpi_errno = MPIR_Err_return_comm( comm_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Sendrecv - Sends and receives a message
Input Parameters:
+ sendbuf - initial address of send buffer (choice)
. sendcount - number of elements in send buffer (integer)
. sendtype - type of elements in send buffer (handle)
. dest - rank of destination (integer)
. sendtag - send tag (integer)
. recvcount - number of elements in receive buffer (integer)
. recvtype - type of elements in receive buffer (handle)
. source - rank of source (integer)
. recvtag - receive tag (integer)
- comm - communicator (handle)
Output Parameters:
+ recvbuf - initial address of receive buffer (choice)
- status - status object (Status). This refers to the receive operation.
.N ThreadSafe
.N Fortran
.N FortranStatus
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_COMM
.N MPI_ERR_COUNT
.N MPI_ERR_TYPE
.N MPI_ERR_TAG
.N MPI_ERR_RANK
@*/
int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
int dest, int sendtag,
void *recvbuf, int recvcount, MPI_Datatype recvtype,
int source, int recvtag, MPI_Comm comm, MPI_Status * status)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Comm *comm_ptr = NULL;
MPIR_Request *sreq = NULL;
MPIR_Request *rreq = NULL;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_SENDRECV);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_THREAD_CS_ENTER(VCI_GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_TERSE_PT2PT_ENTER_BOTH(MPID_STATE_MPI_SENDRECV);
/* Validate handle parameters needing to be converted */
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* Convert handles to MPI objects. */
MPIR_Comm_get_ptr(comm, comm_ptr);
#ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate communicator */
MPIR_Comm_valid_ptr(comm_ptr, mpi_errno, FALSE);
if (mpi_errno)
goto fn_fail;
/* Validate count */
MPIR_ERRTEST_COUNT(sendcount, mpi_errno);
MPIR_ERRTEST_COUNT(recvcount, mpi_errno);
/* Validate status (status_ignore is not the same as null) */
MPIR_ERRTEST_ARGNULL(status, "status", mpi_errno);
/* Validate tags */
MPIR_ERRTEST_SEND_TAG(sendtag, mpi_errno);
MPIR_ERRTEST_RECV_TAG(recvtag, mpi_errno);
/* Validate source and destination */
if (comm_ptr) {
MPIR_ERRTEST_SEND_RANK(comm_ptr, dest, mpi_errno);
MPIR_ERRTEST_RECV_RANK(comm_ptr, source, mpi_errno);
}
/* Validate datatype handles */
MPIR_ERRTEST_DATATYPE(sendtype, "datatype", mpi_errno);
MPIR_ERRTEST_DATATYPE(recvtype, "datatype", mpi_errno);
/* Validate datatype objects */
if (HANDLE_GET_KIND(sendtype) != HANDLE_KIND_BUILTIN) {
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(sendtype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
}
if (HANDLE_GET_KIND(recvtype) != HANDLE_KIND_BUILTIN) {
MPIR_Datatype *datatype_ptr = NULL;
MPIR_Datatype_get_ptr(recvtype, datatype_ptr);
MPIR_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
MPIR_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno)
goto fn_fail;
}
/* Validate buffers */
MPIR_ERRTEST_USERBUFFER(sendbuf, sendcount, sendtype, mpi_errno);
MPIR_ERRTEST_USERBUFFER(recvbuf, recvcount, recvtype, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno =
MPID_Irecv(recvbuf, recvcount, recvtype, source, recvtag, comm_ptr,
MPIR_CONTEXT_INTRA_PT2PT, &rreq);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* FIXME - Performance for small messages might be better if MPID_Send() were used here instead of MPID_Isend() */
mpi_errno =
MPID_Isend(sendbuf, sendcount, sendtype, dest, sendtag, comm_ptr, MPIR_CONTEXT_INTRA_PT2PT,
&sreq);
if (mpi_errno != MPI_SUCCESS) {
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno == MPIX_ERR_NOREQ)
MPIR_ERR_SET(mpi_errno, MPI_ERR_OTHER, "**nomem");
/* FIXME: should we cancel the pending (possibly completed) receive request or wait for it to complete? */
MPIR_Request_free(rreq);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
if (!MPIR_Request_is_complete(sreq) || !MPIR_Request_is_complete(rreq)) {
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (!MPIR_Request_is_complete(sreq) || !MPIR_Request_is_complete(rreq)) {
mpi_errno = MPID_Progress_wait(&progress_state);
if (mpi_errno != MPI_SUCCESS) {
/* --BEGIN ERROR HANDLING-- */
MPID_Progress_end(&progress_state);
goto fn_fail;
/* --END ERROR HANDLING-- */
}
if (unlikely(MPIR_Request_is_anysrc_mismatched(rreq))) {
/* --BEGIN ERROR HANDLING-- */
mpi_errno = MPIR_Request_handle_proc_failed(rreq);
if (!MPIR_Request_is_complete(sreq)) {
MPID_Cancel_send(sreq);
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
}
goto fn_fail;
/* --END ERROR HANDLING-- */
}
}
MPID_Progress_end(&progress_state);
}
mpi_errno = rreq->status.MPI_ERROR;
MPIR_Request_extract_status(rreq, status);
MPIR_Request_free(rreq);
if (mpi_errno == MPI_SUCCESS) {
mpi_errno = sreq->status.MPI_ERROR;
}
MPIR_Request_free(sreq);
if (mpi_errno != MPI_SUCCESS)
goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPIR_FUNC_TERSE_PT2PT_EXIT_BOTH(MPID_STATE_MPI_SENDRECV);
MPID_THREAD_CS_EXIT(VCI_GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
#ifdef HAVE_ERROR_CHECKING
{
mpi_errno =
MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, __func__, __LINE__, MPI_ERR_OTHER,
"**mpi_sendrecv",
"**mpi_sendrecv %p %d %D %i %t %p %d %D %i %t %C %p", sendbuf,
sendcount, sendtype, dest, sendtag, recvbuf, recvcount, recvtype,
source, recvtag, comm, status);
}
#endif
mpi_errno = MPIR_Err_return_comm(comm_ptr, __func__, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
MPID_Request * MPID_Request_create(void)
{
MPID_Request * req;
MPIDI_STATE_DECL(MPID_STATE_MPID_REQUEST_CREATE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_REQUEST_CREATE);
req = MPIU_Handle_obj_alloc(&MPID_Request_mem);
if (req != NULL)
{
MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE,
"allocated request, handle=0x%08x", req->handle);
#ifdef MPICH_DBG_OUTPUT
/*MPIU_Assert(HANDLE_GET_MPI_KIND(req->handle) == MPID_REQUEST);*/
if (HANDLE_GET_MPI_KIND(req->handle) != MPID_REQUEST)
{
int mpi_errno;
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
FCNAME, __LINE__, MPI_ERR_OTHER,
"**invalid_handle", "**invalid_handle %d", req->handle);
MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
}
#endif
/* FIXME: This makes request creation expensive. We need to trim
this to the basics, with additional setup for special-purpose
requests (think base class and inheritance). For example, do we
*really* want to set the kind to UNDEFINED? And should the RMA
values be set only for RMA requests? */
MPIU_Object_set_ref(req, 1);
req->kind = MPID_REQUEST_UNDEFINED;
MPID_cc_set(&req->cc, 1);
req->cc_ptr = &req->cc;
/* FIXME: status fields meaningful only for receive, and even then
should not need to be set. */
req->status.MPI_SOURCE = MPI_UNDEFINED;
req->status.MPI_TAG = MPI_UNDEFINED;
req->status.MPI_ERROR = MPI_SUCCESS;
MPIR_STATUS_SET_COUNT(req->status, 0);
MPIR_STATUS_SET_CANCEL_BIT(req->status, FALSE);
req->comm = NULL;
req->greq_fns = NULL;
req->errflag = MPIR_ERR_NONE;
req->dev.datatype_ptr = NULL;
req->dev.segment_ptr = NULL;
/* Masks and flags for channel device state in an MPID_Request */
req->dev.state = 0;
req->dev.cancel_pending = FALSE;
/* FIXME: RMA ops shouldn't need to be set except when creating a
request for RMA operations */
req->dev.target_win_handle = MPI_WIN_NULL;
req->dev.source_win_handle = MPI_WIN_NULL;
req->dev.lock_queue_entry = NULL;
req->dev.dtype_info = NULL;
req->dev.dataloop = NULL;
req->dev.iov_offset = 0;
req->dev.flags = MPIDI_CH3_PKT_FLAG_NONE;
req->dev.resp_request_handle = MPI_REQUEST_NULL;
req->dev.user_buf = NULL;
req->dev.OnDataAvail = NULL;
req->dev.OnFinal = NULL;
req->dev.user_buf = NULL;
req->dev.drop_data = FALSE;
req->dev.stream_offset = 0;
#ifdef MPIDI_CH3_REQUEST_INIT
MPIDI_CH3_REQUEST_INIT(req);
#endif
}
else
{
/* FIXME: This fails to fail if debugging is turned off */
MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"unable to allocate a request");
}
MPIDI_FUNC_EXIT(MPID_STATE_MPID_REQUEST_CREATE);
return req;
}
PMPI_LOCAL int MPIR_Ibsend_query(void *extra, MPI_Status * status)
{
ibsend_req_info *ibsend_info = (ibsend_req_info *) extra;
MPIR_STATUS_SET_CANCEL_BIT(*status, ibsend_info->cancelled);
return MPI_SUCCESS;
}
/* Complete a request, saving the status data if necessary.
"active" has meaning only if the request is a persistent request; this
allows the completion routines to indicate that a persistent request
was inactive and did not require any extra completion operation.
If debugger information is being provided for pending (user-initiated)
send operations, the macros MPIR_SENDQ_FORGET will be defined to
call the routine MPIR_Sendq_forget; otherwise that macro will be a no-op.
The implementation of the MPIR_Sendq_xxx is in src/mpi/debugger/dbginit.c .
*/
int MPIR_Request_complete(MPI_Request * request, MPID_Request * request_ptr,
MPI_Status * status, int * active)
{
int mpi_errno = MPI_SUCCESS;
*active = TRUE;
switch(request_ptr->kind)
{
case MPID_REQUEST_SEND:
{
if (status != MPI_STATUS_IGNORE)
{
MPIR_STATUS_SET_CANCEL_BIT(*status, MPIR_STATUS_GET_CANCEL_BIT(request_ptr->status));
}
mpi_errno = request_ptr->status.MPI_ERROR;
MPIR_SENDQ_FORGET(request_ptr);
MPID_Request_release(request_ptr);
if (NULL != request) *request = MPI_REQUEST_NULL;
break;
}
case MPID_REQUEST_RECV:
{
MPIR_Request_extract_status(request_ptr, status);
mpi_errno = request_ptr->status.MPI_ERROR;
MPID_Request_release(request_ptr);
if (NULL != request) *request = MPI_REQUEST_NULL;
break;
}
case MPID_PREQUEST_SEND:
{
if (request_ptr->partner_request != NULL)
{
MPID_Request * prequest_ptr = request_ptr->partner_request;
/* reset persistent request to inactive state */
MPID_cc_set(&request_ptr->cc, 0);
request_ptr->cc_ptr = &request_ptr->cc;
request_ptr->partner_request = NULL;
if (prequest_ptr->kind != MPID_UREQUEST)
{
if (status != MPI_STATUS_IGNORE)
{
MPIR_STATUS_SET_CANCEL_BIT(*status, MPIR_STATUS_GET_CANCEL_BIT(prequest_ptr->status));
}
mpi_errno = prequest_ptr->status.MPI_ERROR;
}
else
{
/* This is needed for persistent Bsend requests */
int rc;
rc = MPIR_Grequest_query(prequest_ptr);
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = rc;
}
if (status != MPI_STATUS_IGNORE)
{
MPIR_STATUS_SET_CANCEL_BIT(*status, MPIR_STATUS_GET_CANCEL_BIT(prequest_ptr->status));
}
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = prequest_ptr->status.MPI_ERROR;
}
rc = MPIR_Grequest_free(prequest_ptr);
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = rc;
}
}
MPID_Request_release(prequest_ptr);
}
else
{
if (request_ptr->status.MPI_ERROR != MPI_SUCCESS)
{
/* if the persistent request failed to start then make the
error code available */
if (status != MPI_STATUS_IGNORE)
{
MPIR_STATUS_SET_CANCEL_BIT(*status, FALSE);
}
mpi_errno = request_ptr->status.MPI_ERROR;
}
else
{
MPIR_Status_set_empty(status);
*active = FALSE;
}
}
break;
}
case MPID_PREQUEST_RECV:
{
if (request_ptr->partner_request != NULL)
{
MPID_Request * prequest_ptr = request_ptr->partner_request;
/* reset persistent request to inactive state */
MPID_cc_set(&request_ptr->cc, 0);
request_ptr->cc_ptr = &request_ptr->cc;
request_ptr->partner_request = NULL;
MPIR_Request_extract_status(prequest_ptr, status);
mpi_errno = prequest_ptr->status.MPI_ERROR;
MPID_Request_release(prequest_ptr);
}
else
{
MPIR_Status_set_empty(status);
/* --BEGIN ERROR HANDLING-- */
if (request_ptr->status.MPI_ERROR != MPI_SUCCESS)
{
/* if the persistent request failed to start then make the
error code available */
mpi_errno = request_ptr->status.MPI_ERROR;
}
else
{
*active = FALSE;
}
/* --END ERROR HANDLING-- */
}
break;
}
case MPID_UREQUEST:
{
int rc;
rc = MPIR_Grequest_query(request_ptr);
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = rc;
}
MPIR_Request_extract_status(request_ptr, status);
rc = MPIR_Grequest_free(request_ptr);
if (mpi_errno == MPI_SUCCESS)
{
mpi_errno = rc;
}
MPID_Request_release(request_ptr);
if (NULL != request) *request = MPI_REQUEST_NULL;
break;
}
case MPID_COLL_REQUEST:
case MPID_WIN_REQUEST:
{
mpi_errno = request_ptr->status.MPI_ERROR;
MPIR_Request_extract_status(request_ptr, status);
MPID_Request_release(request_ptr);
if (NULL != request) *request = MPI_REQUEST_NULL;
break;
}
default:
{
/* --BEGIN ERROR HANDLING-- */
/* This should not happen */
MPIR_ERR_SETANDSTMT1(mpi_errno, MPI_ERR_INTERN,;, "**badcase",
"**badcase %d", request_ptr->kind);
break;
/* --END ERROR HANDLING-- */
}
}
int MPID_Cancel_send(MPIR_Request * sreq)
{
MPIDI_VC_t * vc;
int proto;
int flag;
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_CANCEL_SEND);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_CANCEL_SEND);
MPIR_Assert(sreq->kind == MPIR_REQUEST_KIND__SEND);
MPIDI_Request_cancel_pending(sreq, &flag);
if (flag)
{
goto fn_exit;
}
/*
* FIXME: user requests returned by MPI_Ibsend() have a NULL comm pointer
* and no pointer to the underlying communication
* request. For now, we simply fail to cancel the request. In the future,
* we should add a new request kind to indicate that
* the request is a BSEND. Then we can properly cancel the request, much
* in the way we do persistent requests.
*/
if (sreq->comm == NULL)
{
goto fn_exit;
}
MPIDI_Comm_get_vc_set_active(sreq->comm, sreq->dev.match.parts.rank, &vc);
proto = MPIDI_Request_get_msg_type(sreq);
if (proto == MPIDI_REQUEST_SELF_MSG)
{
MPIR_Request * rreq;
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,
"attempting to cancel message sent to self");
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
rreq = MPIDI_CH3U_Recvq_FDU(sreq->handle, &sreq->dev.match);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_MSGQ_MUTEX);
if (rreq)
{
MPIR_Assert(rreq->dev.partner_request == sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"send-to-self cancellation successful, sreq=0x%08x, rreq=0x%08x",
sreq->handle, rreq->handle));
/* Pull the message out of the unexpected queue since it's
* being cancelled. The below request release drops one
* reference. We explicitly drop a second reference,
* because the receive request will never be visible to
* the user. */
MPIR_Request_free(rreq);
MPIR_Request_free(rreq);
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else
{
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"send-to-self cancellation failed, sreq=0x%08x, rreq=0x%08x",
sreq->handle, rreq->handle));
}
goto fn_exit;
}
/* If the message went over a netmod and it provides a cancel_send
function, call it here. */
#ifdef ENABLE_COMM_OVERRIDES
if (vc->comm_ops && vc->comm_ops->cancel_send)
{
mpi_errno = vc->comm_ops->cancel_send(vc, sreq);
goto fn_exit;
}
#endif
/* Check to see if the send is still in the send queue. If so, remove it,
mark the request and cancelled and complete, and
release the device's reference to the request object.
*/
{
int cancelled;
if (proto == MPIDI_REQUEST_RNDV_MSG)
{
MPIR_Request * rts_sreq;
/* The cancellation of the RTS request needs to be atomic through
the destruction of the RTS request to avoid
conflict with release of the RTS request if the CTS is received
(see handling of a rendezvous CTS packet in
MPIDI_CH3U_Handle_recv_pkt()).
MPID_Request_fetch_and_clear_rts_sreq() is used to gurantee
that atomicity. */
MPIDI_Request_fetch_and_clear_rts_sreq(sreq, &rts_sreq);
if (rts_sreq != NULL)
{
cancelled = FALSE;
/* since we attempted to cancel a RTS request, then we are
responsible for releasing that request */
MPIR_Request_free(rts_sreq);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, __func__, __LINE__, MPI_ERR_OTHER,
"**ch3|cancelrndv", 0);
goto fn_exit;
}
/* --END ERROR HANDLING-- */
if (cancelled)
{
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* no other thread should be waiting on sreq, so it is
safe to reset ref_count and cc */
MPIR_cc_set(&sreq->cc, 0);
/* FIXME should be a decr and assert, not a set */
MPIR_Object_set_ref(sreq, 1);
goto fn_exit;
}
}
}
else
{
cancelled = FALSE;
if (cancelled)
{
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* no other thread should be waiting on sreq, so it is safe to
reset ref_count and cc */
MPIR_cc_set(&sreq->cc, 0);
/* FIXME should be a decr and assert, not a set */
MPIR_Object_set_ref(sreq, 1);
goto fn_exit;
}
}
}
/* Part or all of the message has already been sent, so we need to send a
cancellation request to the receiver in an attempt
to catch the message before it is matched. */
{
int was_incomplete;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_cancel_send_req_t * const csr_pkt = &upkt.cancel_send_req;
MPIR_Request * csr_sreq;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"sending cancel request to %d for 0x%08x",
sreq->dev.match.parts.rank, sreq->handle));
/* The completion counter and reference count are incremented to keep
the request around long enough to receive a
response regardless of what the user does (free the request before
waiting, etc.). */
MPIDI_CH3U_Request_increment_cc(sreq, &was_incomplete);
if (!was_incomplete)
{
/* The reference count is incremented only if the request was
complete before the increment. */
MPIR_Request_add_ref( sreq );
}
MPIDI_Pkt_init(csr_pkt, MPIDI_CH3_PKT_CANCEL_SEND_REQ);
csr_pkt->match.parts.rank = sreq->comm->rank;
csr_pkt->match.parts.tag = sreq->dev.match.parts.tag;
csr_pkt->match.parts.context_id = sreq->dev.match.parts.context_id;
csr_pkt->sender_req_id = sreq->handle;
MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex);
mpi_errno = MPIDI_CH3_iStartMsg(vc, csr_pkt, sizeof(*csr_pkt), &csr_sreq);
MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**ch3|cancelreq");
}
if (csr_sreq != NULL)
{
MPIR_Request_free(csr_sreq);
}
}
/* FIXME: if send cancellation packets are allowed to arrive out-of-order
with respect to send packets, then we need to
timestamp send and cancel packets to insure that a cancellation request
does not bypass the send packet to be cancelled
and erroneously cancel a previously sent message with the same request
handle. */
/* FIXME: A timestamp is more than is necessary; a message sequence number
should be adequate. */
fn_fail:
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_CANCEL_SEND);
return mpi_errno;
}
int MPID_nem_ofi_poll(int in_blocking_poll)
{
int complete = 0, mpi_errno = MPI_SUCCESS;
ssize_t ret;
cq_tagged_entry_t wc;
cq_err_entry_t error;
MPIDI_VC_t *vc;
MPID_Request *req;
req_fn reqFn;
BEGIN_FUNC(FCNAME);
do {
/* ----------------------------------------------------- */
/* Poll the completion queue */
/* The strategy here is */
/* ret>0 successfull poll, events returned */
/* ret==0 empty poll, no events/no error */
/* ret<0, error, but some error instances should not */
/* cause MPI to terminate */
/* ----------------------------------------------------- */
ret = fi_cq_read(gl_data.cq, /* Tagged completion queue */
(void *) &wc, /* OUT: Tagged completion entry */
1); /* Number of entries to poll */
if (ret > 0) {
if (NULL != wc.op_context) {
req = context_to_req(wc.op_context);
if (REQ_OFI(req)->event_callback) {
MPIDI_CH3I_NM_OFI_RC(REQ_OFI(req)->event_callback(&wc, req));
continue;
}
reqFn = req->dev.OnDataAvail;
if (reqFn) {
if (REQ_OFI(req)->pack_buffer) {
MPIU_Free(REQ_OFI(req)->pack_buffer);
}
vc = REQ_OFI(req)->vc;
complete = 0;
MPIDI_CH3I_NM_OFI_RC(reqFn(vc, req, &complete));
continue;
}
else {
MPIU_Assert(0);
}
}
else {
MPIU_Assert(0);
}
}
else if (ret == -FI_EAGAIN)
;
else if (ret < 0) {
if (ret == -FI_EAVAIL) {
ret = fi_cq_readerr(gl_data.cq, (void *) &error, 0);
if (error.err == FI_ETRUNC) {
/* ----------------------------------------------------- */
/* This error message should only be delivered on send */
/* events. We want to ignore truncation errors */
/* on the sender side, but complete the request anyway */
/* Other kinds of requests, this is fatal. */
/* ----------------------------------------------------- */
req = context_to_req(error.op_context);
if (req->kind == MPID_REQUEST_SEND) {
mpi_errno = REQ_OFI(req)->event_callback(NULL, req);
}
else if (req->kind == MPID_REQUEST_RECV) {
mpi_errno = REQ_OFI(req)->event_callback(&wc, req);
req->status.MPI_ERROR = MPI_ERR_TRUNCATE;
req->status.MPI_TAG = error.tag;
}
else {
mpi_errno = MPI_ERR_OTHER;
}
}
else if (error.err == FI_ECANCELED) {
req = context_to_req(error.op_context);
MPIR_STATUS_SET_CANCEL_BIT(req->status, TRUE);
}
else {
mpi_errno = MPI_ERR_OTHER;
}
}
else {
MPIR_ERR_CHKANDJUMP4(1, mpi_errno, MPI_ERR_OTHER, "**ofi_poll",
"**ofi_poll %s %d %s %s", __SHORT_FILE__,
__LINE__, FCNAME, fi_strerror(-ret));
}
}
} while (in_blocking_poll && (ret > 0));
END_FUNC_RC(FCNAME);
}
