int MPIDI_CH3_PktHandler_EagerSyncAck( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp )
{
MPIDI_CH3_Pkt_eager_sync_ack_t * esa_pkt = &pkt->eager_sync_ack;
MPID_Request * sreq;
int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG_P(CH3_OTHER,VERBOSE,
"received eager sync ack pkt, sreq=0x%08x", esa_pkt->sender_req_id);
MPID_Request_get_ptr(esa_pkt->sender_req_id, sreq);
/* decrement CC (but don't mark data transfer as complete since the
transfer could still be in progress) */
/* FIXME: This sometimes segfaults */
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*buflen = sizeof(MPIDI_CH3_Pkt_t);
*rreqp = NULL;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_Rendezvous_rput_finish(MPIDI_VC_t * vc,
MPIDI_CH3_Pkt_rput_finish_t * rf_pkt)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *rreq;
int complete;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_RNDV_RPUT_FINISH);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_RNDV_RPUT_FINISH);
MPID_Request_get_ptr(rf_pkt->receiver_req_id, rreq);
if (!MPIDI_CH3I_MRAIL_Finish_request(rreq))
{
return MPI_SUCCESS;
}
if (rreq->mrail.rndv_buf_alloc == 1){
MPIDI_CH3_Rendezvous_unpack_data(vc, rreq);
} else {
rreq->mrail.rndv_buf = NULL;
}
#if defined(CKPT)
MPIDI_CH3I_CR_req_dequeue(rreq);
#endif /* defined(CKPT) */
MPIDI_CH3I_MRAILI_RREQ_RNDV_FINISH(rreq);
mpi_errno = MPIDI_CH3U_Handle_recv_req(vc, rreq, &complete);
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno =
MPIR_Err_create_code(mpi_errno,
MPIR_ERR_RECOVERABLE, FCNAME,
__LINE__, MPI_ERR_OTHER, "**fail", 0);
}
if (complete)
{
vc->ch.recv_active = NULL;
}
else
{
mpi_errno =
MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL,
FCNAME, __LINE__,
MPI_ERR_OTHER, "**fail", 0);
goto fn_exit;
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_RNDV_RPUT_FINISH);
return mpi_errno;
}
int MPIDI_CH3_Rendezvous_rget_send_finish(MPIDI_VC_t * vc,
MPIDI_CH3_Pkt_rget_finish_t *rget_pkt)
{
int mpi_errno = MPI_SUCCESS;
int complete;
MPID_Request *sreq;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_RNDV_RGET_SEND_FINISH);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_RNDV_RGET_SEND_FINISH);
MPID_Request_get_ptr(rget_pkt->sender_req_id, sreq);
if (!MPIDI_CH3I_MRAIL_Finish_request(sreq)) {
return MPI_SUCCESS;
}
MPIDI_CH3I_MRAILI_RREQ_RNDV_FINISH(sreq);
#if 0
if(MPIDI_CH3I_RDMA_Process.has_hsam &&
((req->mrail.rndv_buf_sz > rdma_large_msg_rail_sharing_threshold))) {
/* Adjust the weights of different paths according to the
* timings obtained for the stripes */
adjust_weights(v->vc, req->mrail.stripe_start_time,
req->mrail.stripe_finish_time, req->mrail.initial_weight);
}
#endif
MPIDI_CH3U_Handle_send_req(vc, sreq, &complete);
if (complete != TRUE)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno,
MPIR_ERR_FATAL,
FCNAME,
__LINE__,
MPI_ERR_OTHER,
"**fail",
0);
goto fn_exit;
}
#if defined(CKPT)
MPIDI_CH3I_CR_req_dequeue(sreq);
#endif /* defined(CKPT) */
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_RNDV_RGET_SEND_FINISH);
return mpi_errno;
}
int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp )
{
MPIDI_CH3_Pkt_rndv_send_t * rs_pkt = &pkt->rndv_send;
int mpi_errno = MPI_SUCCESS;
int complete;
char *data_buf;
MPIDI_msg_sz_t data_len;
MPID_Request *req;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"received rndv send (data) pkt");
MPID_Request_get_ptr(rs_pkt->receiver_req_id, req);
data_len = ((*buflen - sizeof(MPIDI_CH3_Pkt_t) >= req->dev.recv_data_sz)
? req->dev.recv_data_sz : *buflen - sizeof(MPIDI_CH3_Pkt_t));
data_buf = (char *)pkt + sizeof(MPIDI_CH3_Pkt_t);
if (req->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len,
&complete);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_RNDV_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else
{
*rreqp = req;
}
}
fn_fail:
return mpi_errno;
}
/*@
MPI_Cancel - Cancels a communication request
Input Parameters:
. request - communication request (handle)
Notes:
The primary expected use of 'MPI_Cancel' is in multi-buffering
schemes, where speculative 'MPI_Irecvs' are made. When the computation
completes, some of these receive requests may remain; using 'MPI_Cancel' allows
the user to cancel these unsatisfied requests.
Cancelling a send operation is much more difficult, in large part because the
send will usually be at least partially complete (the information on the tag,
size, and source are usually sent immediately to the destination).
Users are
advised that cancelling a send, while a local operation (as defined by the MPI
standard), is likely to be expensive (usually generating one or more internal
messages).
.N ThreadSafe
.N Fortran
.N NULL
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_REQUEST
.N MPI_ERR_ARG
@*/
int MPI_Cancel(MPI_Request *request)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request * request_ptr;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_CANCEL);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_PT2PT_FUNC_ENTER(MPID_STATE_MPI_CANCEL);
/* Convert MPI object handles to object pointers */
MPID_Request_get_ptr( *request, request_ptr );
/* Validate parameters if error checking is enabled */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate request_ptr */
MPID_Request_valid_ptr( request_ptr, mpi_errno );
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPIR_Cancel_impl(request_ptr);
if (mpi_errno) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_PT2PT_FUNC_EXIT(MPID_STATE_MPI_CANCEL);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE,
FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_cancel",
"**mpi_cancel %p", request);
}
# endif
mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIR_Test_impl(MPI_Request *request, int *flag, MPI_Status *status)
{
int mpi_errno = MPI_SUCCESS;
int active_flag;
MPID_Request *request_ptr = NULL;
/* If this is a null request handle, then return an empty status */
if (*request == MPI_REQUEST_NULL) {
MPIR_Status_set_empty(status);
*flag = TRUE;
goto fn_exit;
}
*flag = FALSE;
MPID_Request_get_ptr( *request, request_ptr );
/* If the request is already completed AND we want to avoid calling
the progress engine, we could make the call to MPID_Progress_test
conditional on the request not being completed. */
mpi_errno = MPID_Progress_test();
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
if (request_ptr->kind == MPID_UREQUEST &&
request_ptr->greq_fns != NULL &&
request_ptr->greq_fns->poll_fn != NULL)
{
mpi_errno = (request_ptr->greq_fns->poll_fn)(request_ptr->greq_fns->grequest_extra_state, status);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (MPID_Request_is_complete(request_ptr)) {
mpi_errno = MPIR_Request_complete(request, request_ptr, status,
&active_flag);
*flag = TRUE;
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Fall through to the exit */
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_ptl_pkt_cancel_send_resp_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq;
MPIDI_nem_ptl_pkt_cancel_send_resp_t *resp_pkt = (MPIDI_nem_ptl_pkt_cancel_send_resp_t *)pkt;
int i, ret;
MPID_Request_get_ptr(resp_pkt->sender_req_id, sreq);
if (resp_pkt->ack) {
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, TRUE);
/* remove/free any remaining get MEs and handles */
for (i = 0; i < REQ_PTL(sreq)->num_gets; i++) {
ret = PtlMEUnlink(REQ_PTL(sreq)->get_me_p[i]);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeunlink", "**ptlmeunlink %s", MPID_nem_ptl_strerror(ret));
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
if (REQ_PTL(sreq)->get_me_p)
MPIU_Free(REQ_PTL(sreq)->get_me_p);
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"message cancelled");
} else {
MPIR_STATUS_SET_CANCEL_BIT(sreq->status, FALSE);
MPIU_DBG_MSG(CH3_OTHER,TYPICAL,"unable to cancel message");
}
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPI_Start - Initiates a communication with a persistent request handle
Input Parameters:
. request - communication request (handle)
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_REQUEST
@*/
int MPI_Start(MPI_Request *request)
{
static const char FCNAME[] = "MPI_Start";
MPID_Request * request_ptr = NULL;
int mpi_errno = MPI_SUCCESS;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_START);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_PT2PT_FUNC_ENTER(MPID_STATE_MPI_START);
/* Validate handle parameters needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_ARGNULL(request,"request",mpi_errno);
MPIR_ERRTEST_REQUEST(*request, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* Convert MPI request handle to a request object pointer */
MPID_Request_get_ptr( *request, request_ptr );
/* Validate object pointers if error checking is enabled */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPID_Request_valid_ptr( request_ptr, mpi_errno );
if (mpi_errno) goto fn_fail;
MPIR_ERRTEST_PERSISTENT(request_ptr, mpi_errno);
MPIR_ERRTEST_PERSISTENT_ACTIVE(request_ptr, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPID_Startall(1, &request_ptr);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_PT2PT_FUNC_EXIT(MPID_STATE_MPI_START);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_start", "**mpi_start %p", request);
}
# endif
mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
static inline int check_window_state(MPID_Win * win_ptr, int *made_progress)
{
MPID_Request *fence_req_ptr = NULL;
int i, mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_CHECK_WINDOW_STATE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_CHECK_WINDOW_STATE);
(*made_progress) = 0;
switch (win_ptr->states.access_state) {
case MPIDI_RMA_FENCE_ISSUED:
MPID_Request_get_ptr(win_ptr->fence_sync_req, fence_req_ptr);
if (MPID_Request_is_complete(fence_req_ptr)) {
win_ptr->states.access_state = MPIDI_RMA_FENCE_GRANTED;
MPID_Request_release(fence_req_ptr);
win_ptr->fence_sync_req = MPI_REQUEST_NULL;
num_active_issued_win--;
MPIU_Assert(num_active_issued_win >= 0);
(*made_progress) = 1;
}
break;
case MPIDI_RMA_PSCW_ISSUED:
if (win_ptr->start_req == NULL) {
/* for MPI_MODE_NOCHECK and all targets on SHM,
* we do not create PSCW requests on window. */
win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
num_active_issued_win--;
MPIU_Assert(num_active_issued_win >= 0);
(*made_progress) = 1;
}
else {
for (i = 0; i < win_ptr->start_grp_size; i++) {
MPID_Request *start_req_ptr = NULL;
if (win_ptr->start_req[i] == MPI_REQUEST_NULL)
continue;
MPID_Request_get_ptr(win_ptr->start_req[i], start_req_ptr);
if (MPID_Request_is_complete(start_req_ptr)) {
MPID_Request_release(start_req_ptr);
win_ptr->start_req[i] = MPI_REQUEST_NULL;
}
else {
break;
}
}
if (i == win_ptr->start_grp_size) {
win_ptr->states.access_state = MPIDI_RMA_PSCW_GRANTED;
num_active_issued_win--;
MPIU_Assert(num_active_issued_win >= 0);
(*made_progress) = 1;
MPIU_Free(win_ptr->start_req);
win_ptr->start_req = NULL;
}
}
break;
case MPIDI_RMA_LOCK_ALL_ISSUED:
if (win_ptr->outstanding_locks == 0) {
win_ptr->states.access_state = MPIDI_RMA_LOCK_ALL_GRANTED;
(*made_progress) = 1;
}
break;
default:
break;
} /* end of switch */
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_CHECK_WINDOW_STATE);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Test - Tests for the completion of a request
Input Parameters:
. request - MPI request (handle)
Output Parameters:
+ flag - true if operation completed (logical)
- status - status object (Status). May be 'MPI_STATUS_IGNORE'.
.N ThreadSafe
.N waitstatus
.N Fortran
.N FortranStatus
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_REQUEST
.N MPI_ERR_ARG
@*/
int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *request_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_TEST);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_PT2PT_FUNC_ENTER(MPID_STATE_MPI_TEST);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_ARGNULL(request, "request", mpi_errno);
MPIR_ERRTEST_REQUEST_OR_NULL(*request, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
MPID_Request_get_ptr( *request, request_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
if (*request != MPI_REQUEST_NULL)
{
/* Validate request_ptr */
MPID_Request_valid_ptr( request_ptr, mpi_errno );
if (mpi_errno) goto fn_fail;
}
MPIR_ERRTEST_ARGNULL(flag, "flag", mpi_errno);
/* NOTE: MPI_STATUS_IGNORE != NULL */
MPIR_ERRTEST_ARGNULL(status, "status", mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPIR_Test_impl(request, flag, status);
if (mpi_errno) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_PT2PT_FUNC_EXIT(MPID_STATE_MPI_TEST);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_test",
"**mpi_test %p %p %p", request, flag, status);
}
# endif
mpi_errno = MPIR_Err_return_comm(request_ptr ? request_ptr->comm : NULL,
FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/**
* FIXME: Ideally the header size should be determined by high level macros,
* instead of hacking the message header at the device layer
*/
int MPIDI_CH3I_nem_ib_parse_header(MPIDI_VC_t * vc,
vbuf * v, void **pkt, int *header_size)
{
void *vstart;
MPIDI_nem_ib_pkt_comm_header *header;
#ifdef CRC_CHECK
unsigned long crc;
#endif
int mpi_errno = MPI_SUCCESS;
int ret;
DEBUG_PRINT("[parse header] vbuf address %p\n", v);
vstart = v->pheader;
header = (MPIDI_nem_ib_pkt_comm_header *)v->iheader;
DEBUG_PRINT("[parse header] header type %d\n", header->type);
/* set it to the header size by default */
*header_size = sizeof(MPIDI_CH3_Pkt_t);
#ifdef CRC_CHECK
crc = update_crc(1, (void *)((uintptr_t)header+sizeof *header),
v->content_size - sizeof *header);
if (crc != header->mrail.crc) {
int rank; PMI_Get_rank(&rank);
MPIU_Error_printf(stderr, "CRC mismatch, get %lx, should be %lx "
"type %d, ocntent size %d\n",
crc, header->mrail.crc, header->type, v->content_size);
exit( EXIT_FAILURE );
}
#endif
switch (header->type) {
/*header caching codes */
#ifndef MV2_DISABLE_HEADER_CACHING
case (MPIDI_CH3_PKT_FAST_EAGER_SEND):
case (MPIDI_CH3_PKT_FAST_EAGER_SEND_WITH_REQ):
{
/* since header caching do not have regular iheader,
* revert back pre-adjust
*/
v->content_size += IB_PKT_HEADER_LENGTH;
vstart -= IB_PKT_HEADER_LENGTH;
v->pheader -= IB_PKT_HEADER_LENGTH;
MPIDI_nem_ib_pkt_fast_eager *fast_header = vstart;
MPIDI_CH3_Pkt_eager_send_t *eager_header =
(MPIDI_CH3_Pkt_eager_send_t *) VC_FIELD(vc, connection)->rfp.
cached_incoming;
MPIDI_nem_ib_pkt_comm_header *eager_iheader =
(MPIDI_nem_ib_pkt_comm_header *) VC_FIELD(vc, connection)->rfp.
cached_incoming_iheader;
if (MPIDI_CH3_PKT_FAST_EAGER_SEND == header->type) {
*header_size = sizeof(MPIDI_nem_ib_pkt_fast_eager);
} else {
*header_size =
sizeof(MPIDI_nem_ib_pkt_fast_eager_with_req);
eager_header->sender_req_id =
((MPIDI_nem_ib_pkt_fast_eager_with_req *)
vstart)->sender_req_id;
}
header = eager_iheader;
DEBUG_PRINT("[receiver side] cached credit %d\n",
eager_iheader->rdma_credit);
eager_header->data_sz = fast_header->bytes_in_pkt;
*pkt = (void *) eager_header;
DEBUG_PRINT
("[recv: parse header] faster headersize returned %d\n",
*header_size);
}
break;
#endif
case (MPIDI_CH3_PKT_EAGER_SEND):
{
DEBUG_PRINT("[recv: parse header] pkt eager send\n");
/* header caching codes */
#ifndef MV2_DISABLE_HEADER_CACHING
if (v->padding != NORMAL_VBUF_FLAG &&
(v->content_size - sizeof(MPIDI_CH3_Pkt_t) <= MAX_SIZE_WITH_HEADER_CACHING) ) {
/* Only cache header if the packet is from RdMA path
* XXXX: what is R3_FLAG?
*/
MPIU_Memcpy((VC_FIELD(vc, connection)->rfp.cached_incoming), vstart,
sizeof(MPIDI_CH3_Pkt_eager_send_t));
MPIU_Memcpy((VC_FIELD(vc, connection)->rfp.cached_incoming_iheader), header,
sizeof(MPIDI_nem_ib_pkt_comm_header));
}
#endif
*pkt = (MPIDI_CH3_Pkt_t *) vstart;
*header_size = sizeof(MPIDI_CH3_Pkt_t);
DEBUG_PRINT("[recv: parse header] headersize returned %d\n",
*header_size);
}
break;
case (MPIDI_CH3_PKT_RNDV_REQ_TO_SEND):
case (MPIDI_CH3_PKT_RNDV_CLR_TO_SEND):
case MPIDI_CH3_PKT_EAGER_SYNC_ACK:
case MPIDI_NEM_PKT_LMT_RTS:
case MPIDI_NEM_PKT_LMT_CTS:
case MPIDI_NEM_PKT_LMT_DONE:
case MPIDI_NEM_PKT_LMT_COOKIE:
/* CKPT codes */
#ifdef CKPT
case MPIDI_CH3_PKT_CM_SUSPEND:
case MPIDI_CH3_PKT_CM_REACTIVATION_DONE:
case MPIDI_CH3_PKT_CR_REMOTE_UPDATE:
#endif
{
*pkt = vstart;
}
break;
case MPIDI_CH3_PKT_CANCEL_SEND_REQ:
{
*pkt = vstart;
/*Fix: Need to unregister and free the rndv buffer in get protocol.*/
}
break;
case MPIDI_CH3_PKT_CANCEL_SEND_RESP:
{
MPID_Request *req;
*pkt = vstart;
MPID_Request_get_ptr(((MPIDI_CH3_Pkt_cancel_send_resp_t *)(*pkt))->sender_req_id, req);
if (req != NULL) {
/* unregister and free the rndv buffer */
MPIDI_NEM_IB_RREQ_RNDV_FINISH(req);
}
}
break;
case (MPIDI_CH3_PKT_NOOP):
{
*pkt = v->iheader;
}
break;
/* rfp codes */
case MPIDI_CH3_PKT_ADDRESS:
{
*pkt = v->iheader;
MPIDI_nem_ib_recv_addr(vc, vstart);
break;
}
case MPIDI_CH3_PKT_ADDRESS_REPLY:
{
*pkt = v->iheader;
MPIDI_nem_ib_recv_addr_reply(vc, vstart);
break;
}
case MPIDI_CH3_PKT_PACKETIZED_SEND_START:
{
*pkt = vstart;
*header_size = sizeof(MPIDI_CH3_Pkt_packetized_send_start_t);
break;
}
case MPIDI_CH3_PKT_PACKETIZED_SEND_DATA:
{
*header_size = sizeof(MPIDI_CH3_Pkt_packetized_send_data_t);
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_RNDV_R3_DATA:
{
*header_size = sizeof(MPIDI_CH3_Pkt_rndv_r3_data_t);
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_RNDV_R3_ACK:
{
*pkt = v->iheader;
MPIDI_nem_ib_lmt_r3_recv_ack(vc, vstart);
break;
}
#if defined(USE_EAGER_SHORT)
case MPIDI_CH3_PKT_EAGERSHORT_SEND:
#endif
case MPIDI_CH3_PKT_EAGER_SYNC_SEND:
case MPIDI_CH3_PKT_READY_SEND:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_PUT:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_GET:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_GET_RESP: /*15 */
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_ACCUMULATE:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_LOCK:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_LOCK_GRANTED:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_PT_RMA_DONE:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_LOCK_PUT_UNLOCK:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_LOCK_GET_UNLOCK:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_LOCK_ACCUM_UNLOCK:
case MPIDI_CH3_PKT_ACCUM_IMMED:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_FLOW_CNTL_UPDATE:
{
*pkt = vstart;
break;
}
case MPIDI_CH3_PKT_CLOSE:
{
*pkt = vstart;
}
break;
default:
{
/* Header is corrupted if control has reached here in prototype */
/* */
MPIU_ERR_SETFATALANDJUMP2(mpi_errno,
MPI_ERR_OTHER,
"**fail",
"**fail %s %d",
"Control shouldn't reach here "
"in prototype, header %d\n",
header->type);
}
}
DEBUG_PRINT("Before set credit, vc: %p, v->rail: %d, "
"pkt: %p, pheader: %p\n", vc, v->rail, pkt, v->pheader);
SET_CREDIT(header, VC_FIELD(vc, connection), (v->rail));
if (VC_FIELD(vc, connection)->srp.credits[v->rail].remote_credit > 0 &&
VC_FIELD(vc, connection)->srp.credits[v->rail].backlog.len > 0) {
/* backlog send codes */
MRAILI_Backlog_send(vc, v->rail);
}
/* if any credits remain, schedule rendezvous progress */
if ((VC_FIELD(vc, connection)->srp.credits[v->rail].remote_credit > 0
/* rfp codes */
|| (VC_FIELD(vc, connection)->rfp.ptail_RDMA_send !=
VC_FIELD(vc, connection)->rfp.phead_RDMA_send)
)
&& (VC_FIELD(vc, connection)->sreq_head != NULL)) {
/* rndv codes */
#if 0
PUSH_FLOWLIST(vc);
#endif
}
/* rfp codes */
if ((VC_FIELD(vc, connection)->rfp.RDMA_recv_buf == NULL) && /*(c->initialized) && */
num_rdma_buffer && !VC_FIELD(vc, connection)->rfp.rdma_failed) {
if ((process_info.polling_group_size + rdma_pending_conn_request) <
rdma_polling_set_limit) {
VC_FIELD(vc, connection)->rfp.eager_start_cnt++;
if (rdma_polling_set_threshold <
VC_FIELD(vc, connection)->rfp.eager_start_cnt) {
{
ret = vbuf_fast_rdma_alloc(vc, 1);
if (ret == MPI_SUCCESS) {
vbuf_address_send(vc);
rdma_pending_conn_request++;
} else {
VC_FIELD(vc, connection)->rfp.rdma_failed = 1;
}
goto fn_exit;
}
}
}
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_Rendezvouz_r3_recv_data(MPIDI_VC_t * vc, vbuf * buffer)
{
int mpi_errno = MPI_SUCCESS;
int skipsize = sizeof(MPIDI_CH3_Pkt_rndv_r3_data_t);
int nb, complete;
MPID_Request *rreq;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_RNDV_R3_RCV_DATA);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_RNDV_R3_RCV_DATA);
MPID_Request_get_ptr(((MPIDI_CH3_Pkt_rndv_r3_data_t *) (buffer->
pheader))->
receiver_req_id, rreq);
if (!(VAPI_PROTOCOL_R3 == rreq->mrail.protocol ||
VAPI_PROTOCOL_RPUT == rreq->mrail.protocol)) {
int rank;
PMI_Get_rank(&rank);
DEBUG_PRINT( "[rank %d]get wrong req protocol, req %p, protocol %d\n", rank,
rreq, rreq->mrail.protocol);
MPIU_Assert(VAPI_PROTOCOL_R3 == rreq->mrail.protocol ||
VAPI_PROTOCOL_RPUT == rreq->mrail.protocol);
}
rreq->mrail.protocol = VAPI_PROTOCOL_R3;
mpi_errno = MPIDI_CH3I_MRAIL_Fill_Request(rreq, buffer, skipsize, &nb);
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno,
MPIR_ERR_FATAL,
FCNAME,
__LINE__,
MPI_ERR_OTHER,
"**fail",
0);
goto fn_exit;
}
vc->ch.received_r3_data += nb;
skipsize += nb;
DEBUG_PRINT("[recv r3: handle read] filled request nb is %d\n", nb);
if (MPIDI_CH3I_Request_adjust_iov(rreq, nb)) {
mpi_errno = MPIDI_CH3U_Handle_recv_req(vc, rreq, &complete);
DEBUG_PRINT("[recv: handle read] adjust req fine, complete %d\n",
complete);
if (mpi_errno != MPI_SUCCESS) {
mpi_errno =
MPIR_Err_create_code(mpi_errno,
MPIR_ERR_RECOVERABLE, FCNAME,
__LINE__, MPI_ERR_OTHER, "**fail", 0);
goto fn_exit;
}
while (complete != TRUE) {
mpi_errno =
MPIDI_CH3I_MRAIL_Fill_Request(rreq, buffer, skipsize, &nb);
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno,
MPIR_ERR_FATAL,
FCNAME,
__LINE__,
MPI_ERR_OTHER,
"**fail",
0);
goto fn_exit;
}
vc->ch.received_r3_data += nb;
if (!MPIDI_CH3I_Request_adjust_iov(rreq, nb)) {
goto fn_exit;
}
skipsize += nb;
mpi_errno = MPIDI_CH3U_Handle_recv_req(vc, rreq, &complete);
DEBUG_PRINT
("[recv: handle read] adjust req fine, complete %d\n",
complete);
if (mpi_errno != MPI_SUCCESS) {
mpi_errno =
MPIR_Err_create_code(mpi_errno,
MPIR_ERR_RECOVERABLE, FCNAME,
__LINE__, MPI_ERR_OTHER, "**fail",
0);
goto fn_exit;
}
}
if (TRUE == complete) {
rreq->mrail.protocol = VAPI_PROTOCOL_RENDEZVOUS_UNSPECIFIED;
}
}
fn_exit:
#ifndef DAPL_DEFAULT_PROVIDER
if (MPIDI_CH3I_RDMA_Process.has_srq) {
if ( vc->ch.received_r3_data >= rdma_max_r3_pending_data) {
DEBUG_PRINT("recved data: %d send ack\n", vc->ch.received_r3_data );
MPIDI_CH3I_MRAILI_Rendezvous_r3_ack_send(vc);
}
}
#endif
DEBUG_PRINT("Successfully return from r3 recv\n");
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_RNDV_R3_RCV_DATA);
return mpi_errno;
}
/*@
MPI_Grequest_complete - Notify MPI that a user-defined request is complete
Input Parameter:
. request - Generalized request to mark as complete
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.seealso: MPI_Grequest_start
@*/
int MPI_Grequest_complete( MPI_Request request )
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *request_ptr;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_GREQUEST_COMPLETE);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_GREQUEST_COMPLETE);
/* Validate handle parameters needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_REQUEST(request, mpi_errno);
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPID_Request_get_ptr( request, request_ptr );
/* Validate parameters if error checking is enabled */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPID_Request_valid_ptr(request_ptr,mpi_errno);
if (request_ptr && request_ptr->kind != MPID_UREQUEST) {
mpi_errno = MPIR_Err_create_code( MPI_SUCCESS, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_ARG,
"**notgenreq", 0 );
}
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
MPIR_Grequest_complete_impl(request_ptr);
/* ... end of body of routine ... */
#ifdef HAVE_ERROR_CHECKING
fn_exit:
#endif
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_GREQUEST_COMPLETE);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
fn_fail:
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_grequest_complete",
"**mpi_grequest_complete %R", request);
}
mpi_errno = MPIR_Err_return_comm( NULL, FCNAME, mpi_errno );
goto fn_exit;
# endif
/* --END ERROR HANDLING-- */
}
/*@
MPI_Imrecv - Nonblocking receive of message matched by MPI_Mprobe or MPI_Improbe.
Input/Output Parameters:
. message - message (handle)
Input Parameters:
+ count - number of elements in the receive buffer (non-negative integer)
- datatype - datatype of each receive buffer element (handle)
Output Parameters:
+ buf - initial address of the receive buffer (choice)
- request - communication request (handle)
.N ThreadSafe
.N Fortran
.N Errors
@*/
int MPI_Imrecv(void *buf, int count, MPI_Datatype datatype, MPI_Message *message, MPI_Request *request)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *rreq = NULL;
MPID_Request *msgp = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_IMRECV);
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_IMRECV);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS
{
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
/* TODO more checks may be appropriate */
}
MPID_END_ERROR_CHECKS
}
# endif /* HAVE_ERROR_CHECKING */
/* Convert MPI object handles to object pointers */
MPID_Request_get_ptr(*message, msgp);
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS
{
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPID_Datatype *datatype_ptr = NULL;
MPID_Datatype_get_ptr(datatype, datatype_ptr);
MPID_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
MPID_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
/* MPI_MESSAGE_NO_PROC should yield a "proc null" status */
if (*message != MPI_MESSAGE_NO_PROC) {
MPID_Request_valid_ptr(msgp, mpi_errno);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP((msgp->kind != MPID_REQUEST_MPROBE),
mpi_errno, MPI_ERR_ARG, "**reqnotmsg");
}
MPIR_ERRTEST_ARGNULL(request, "request", mpi_errno);
/* TODO more checks may be appropriate (counts, in_place, buffer aliasing, etc) */
}
MPID_END_ERROR_CHECKS
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
mpi_errno = MPID_Imrecv(buf, count, datatype, msgp, &rreq);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIU_Assert(rreq != NULL);
*request = rreq->handle;
*message = MPI_MESSAGE_NULL;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_IMRECV);
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,
"**mpi_imrecv", "**mpi_imrecv %p %d %D %p %p", buf, count, datatype, message, request);
}
# endif
mpi_errno = MPIR_Err_return_comm(NULL, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDI_CH3_PktHandler_RndvClrToSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp )
{
MPIDI_CH3_Pkt_rndv_clr_to_send_t * cts_pkt = &pkt->rndv_clr_to_send;
MPID_Request * sreq;
MPID_Request * rts_sreq;
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_rndv_send_t * rs_pkt = &upkt.rndv_send;
int dt_contig;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype * dt_ptr;
int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"received rndv CTS pkt");
MPID_Request_get_ptr(cts_pkt->sender_req_id, sreq);
MPIU_DBG_PRINTF(("received cts, count=%d\n", sreq->dev.user_count));
sreq->dev.OnDataAvail = 0;
sreq->dev.OnFinal = 0;
/* Release the RTS request if one exists.
MPID_Request_fetch_and_clear_rts_sreq() needs to be atomic to
prevent
cancel send from cancelling the wrong (future) request.
If MPID_Request_fetch_and_clear_rts_sreq() returns a NULL
rts_sreq, then MPID_Cancel_send() is responsible for releasing
the RTS request object. */
MPIDI_Request_fetch_and_clear_rts_sreq(sreq, &rts_sreq);
if (rts_sreq != NULL)
{
MPID_Request_release(rts_sreq);
}
*buflen = sizeof(MPIDI_CH3_Pkt_t);
MPIDI_Pkt_init(rs_pkt, MPIDI_CH3_PKT_RNDV_SEND);
rs_pkt->receiver_req_id = cts_pkt->receiver_req_id;
MPIDI_Datatype_get_info(sreq->dev.user_count, sreq->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
if (dt_contig)
{
MPID_IOV iov[MPID_IOV_LIMIT];
MPIU_DBG_MSG_FMT(CH3_OTHER,VERBOSE,(MPIU_DBG_FDEST,
"sending contiguous rndv data, data_sz=" MPIDI_MSG_SZ_FMT,
data_sz));
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)rs_pkt;
iov[0].MPID_IOV_LEN = sizeof(*rs_pkt);
iov[1].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)((char *)sreq->dev.user_buf + dt_true_lb);
iov[1].MPID_IOV_LEN = data_sz;
MPIU_THREAD_CS_ENTER(CH3COMM,vc);
mpi_errno = MPIU_CALL(MPIDI_CH3,iSendv(vc, sreq, iov, 2));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|senddata");
}
else
{
sreq->dev.segment_ptr = MPID_Segment_alloc( );
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
MPIU_THREAD_CS_ENTER(CH3COMM,vc);
mpi_errno = vc->sendNoncontig_fn(vc, sreq, rs_pkt, sizeof(*rs_pkt));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**ch3|senddata");
}
*rreqp = NULL;
fn_fail:
return mpi_errno;
}
int MPID_Startall(int count, MPID_Request * requests[])
{
int i;
int rc;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_STARTALL);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_STARTALL);
for (i = 0; i < count; i++)
{
MPID_Request * const preq = requests[i];
/* FIXME: The odd 7th arg (match.context_id - comm->context_id)
is probably to get the context offset. Do we really need the
context offset? Is there any case where the offset isn't zero? */
switch (MPIDI_Request_get_type(preq))
{
case MPIDI_REQUEST_TYPE_RECV:
{
rc = MPID_Irecv(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->recvcontext_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_SEND:
{
rc = MPID_Isend(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->context_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_RSEND:
{
rc = MPID_Irsend(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->context_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_SSEND:
{
rc = MPID_Issend(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->context_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_BSEND:
{
MPI_Request sreq_handle;
rc = MPIR_Ibsend_impl(preq->dev.user_buf, preq->dev.user_count,
preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm,
&sreq_handle);
if (rc == MPI_SUCCESS)
{
MPID_Request_get_ptr(sreq_handle, preq->partner_request);
}
break;
}
default:
{
/* --BEGIN ERROR HANDLING-- */
rc = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_INTERN, "**ch3|badreqtype",
"**ch3|badreqtype %d", MPIDI_Request_get_type(preq));
/* --END ERROR HANDLING-- */
}
}
if (rc == MPI_SUCCESS)
{
preq->status.MPI_ERROR = MPI_SUCCESS;
preq->cc_ptr = &preq->partner_request->cc;
}
/* --BEGIN ERROR HANDLING-- */
else
{
/* If a failure occurs attempting to start the request, then we
assume that partner request was not created, and stuff
the error code in the persistent request. The wait and test
routines will look at the error code in the persistent
request if a partner request is not present. */
preq->partner_request = NULL;
preq->status.MPI_ERROR = rc;
preq->cc_ptr = &preq->cc;
MPID_cc_set(&preq->cc, 0);
}
/* --END ERROR HANDLING-- */
}
MPIDI_FUNC_EXIT(MPID_STATE_MPID_STARTALL);
return mpi_errno;
}
