static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,"Socket event of type %d", event->op_type );
switch (event->op_type)
{
case MPIDU_SOCK_OP_READ:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
MPID_Request * rreq = conn->recv_active;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS)
{
/* FIXME: the following should be handled by the close
protocol */
if (MPIR_ERR_GET_CLASS(event->error) != MPIDU_SOCK_ERR_CONN_CLOSED) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
break;
}
/* --END ERROR HANDLING-- */
if (conn->state == CONN_STATE_CONNECTED)
{
if (conn->recv_active == NULL)
{
MPIDI_msg_sz_t buflen = sizeof (MPIDI_CH3_Pkt_t);
MPIU_Assert(conn->pkt.type < MPIDI_CH3_PKT_END_CH3);
mpi_errno = pktArray[conn->pkt.type]( conn->vc, &conn->pkt,
&buflen, &rreq );
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
MPIU_Assert(buflen == sizeof (MPIDI_CH3_Pkt_t));
if (rreq == NULL)
{
if (conn->state != CONN_STATE_CLOSING)
{
/* conn->recv_active = NULL; --
already set to NULL */
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
}
else
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
else /* incoming data */
{
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(rreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, rreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
conn->recv_active = NULL;
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to be read */
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
}
else if (conn->state == CONN_STATE_OPEN_LRECV_DATA)
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connopen_event( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
else /* Handling some internal connection establishment or
tear down packet */
{
mpi_errno = MPIDI_CH3_Sockconn_handle_conn_event( conn );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
break;
}
/* END OF SOCK_OP_READ */
case MPIDU_SOCK_OP_WRITE:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
/* --END ERROR HANDLING-- */
if (conn->send_active)
{
MPID_Request * sreq = conn->send_active;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to send */
{
for(;;)
{
MPID_IOV * iovp;
MPIU_Size_t nb;
iovp = sreq->dev.iov;
mpi_errno = MPIDU_Sock_writev(conn->sock, iovp, sreq->dev.iov_count, &nb);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|immedwrite", "ch3|sock|immedwrite %p %p %p",
sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"immediate writev, vc=%p, sreq=0x%08x, nb=" MPIDI_MSG_SZ_FMT,
conn->vc, sreq->handle, nb));
if (nb > 0 && adjust_iov(&iovp, &sreq->dev.iov_count, nb))
{
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
break;
}
}
else
{
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=%p, conn=%p, sreq=0x%08x",
conn->vc, conn, sreq->handle));
mpi_errno = MPIDU_Sock_post_writev(conn->sock, iovp, sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postwrite",
"ch3|sock|postwrite %p %p %p", sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
break;
}
}
}
}
else /* finished writing internal packet header */
{
/* the connection is not active yet */
mpi_errno = MPIDI_CH3_Sockconn_handle_connwrite( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
break;
}
/* END OF SOCK_OP_WRITE */
case MPIDU_SOCK_OP_ACCEPT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_accept_event();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CONNECT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connect_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr,
event->error );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CLOSE:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_close_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_WAKEUP:
{
MPIDI_CH3_Progress_signal_completion();
/* MPIDI_CH3I_progress_completion_count++; */
break;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_iStartMsgv(MPIDI_VC_t * vc, MPID_IOV * iov, int n_iov,
MPID_Request ** sreq_ptr)
{
MPID_Request * sreq = NULL;
MPIDI_CH3I_VC *vcch = &vc->ch;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_ISTARTMSGV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_ISTARTMSGV);
MPIU_Assert( n_iov <= MPID_IOV_LIMIT);
/* The SOCK channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
MPIU_DBG_STMT(CH3_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t*)iov[0].MPID_IOV_BUF));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
int rc;
MPIU_Size_t nb;
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPIU_DBG_PKT(vcch->conn,(MPIDI_CH3_Pkt_t*)iov[0].MPID_IOV_BUF,"isend");
/* MT - need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
also try to write */
rc = MPIDU_Sock_writev(vcch->sock, iov, n_iov, &nb);
if (rc == MPI_SUCCESS)
{
int offset = 0;
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
while (offset < n_iov)
{
if (nb >= (int)iov[offset].MPID_IOV_LEN)
{
nb -= iov[offset].MPID_IOV_LEN;
offset++;
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"partial write, request enqueued at head");
sreq = create_request(iov, n_iov, offset, nb);
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(vcch->conn->sock, sreq->dev.iov + offset,
sreq->dev.iov_count - offset, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
break;
}
}
if (offset == n_iov)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"entire write complete");
}
}
/* --BEGIN ERROR HANDLING-- */
else
{
MPIU_DBG_MSG_D(CH3_CHANNEL,TYPICAL,
"ERROR - MPIDU_Sock_writev failed, rc=%d", rc);
sreq = MPID_Request_create();
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
sreq->kind = MPID_REQUEST_SEND;
MPID_cc_set(&(sreq->cc), 0);
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send in progress, request enqueued");
sreq = create_request(iov, n_iov, 0, 0);
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING)
{
MPIU_DBG_VCUSE(vc,
"connecting. enqueuing request");
/* queue the data so it can be sent after the connection is formed */
sreq = create_request(iov, n_iov, 0, 0);
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED)
{
MPIU_DBG_VCUSE(vc,
"unconnected. posting connect and enqueuing request");
/* queue the data so it can be sent after the connection is formed */
sreq = create_request(iov, n_iov, 0, 0);
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
/* Form a new connection */
MPIDI_CH3I_VC_post_connect(vc);
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPIU_DBG_VCUSE(vc,"forming connection, request enqueued");
sreq = create_request(iov, n_iov, 0, 0);
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
/* Connection failed, so allocate a request and return an error. */
MPIU_DBG_VCUSE(vc,"ERROR - connection failed");
sreq = MPID_Request_create();
if (sreq == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
sreq->kind = MPID_REQUEST_SEND;
MPID_cc_set(&(sreq->cc), 0);
sreq->status.MPI_ERROR = MPIR_Err_create_code( MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|connectionfailed",0 );
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
fn_fail:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_ISTARTMSGV);
return mpi_errno;
}
int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
MPID_IOV * iov, int n_iov)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vcch = &vc->ch;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_ISENDV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_ISENDV);
MPIU_Assert(n_iov <= MPID_IOV_LIMIT);
MPIU_Assert(iov[0].MPID_IOV_LEN <= sizeof(MPIDI_CH3_Pkt_t));
/* The sock channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
MPIU_DBG_STMT(CH3_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)iov[0].MPID_IOV_BUF));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
MPIU_Size_t nb;
int rc;
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPIU_DBG_PKT(vcch->conn,(MPIDI_CH3_Pkt_t*)iov[0].MPID_IOV_BUF,
"isendv");
/* MT - need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
also try to write */
/* FIXME: the current code only agressively writes the first IOV.
Eventually it should be changed to agressively write
as much as possible. Ideally, the code would be shared between
the send routines and the progress engine. */
rc = MPIDU_Sock_writev(vcch->sock, iov, n_iov, &nb);
if (rc == MPI_SUCCESS)
{
int offset = 0;
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
while (offset < n_iov)
{
if ((int)iov[offset].MPID_IOV_LEN <= nb)
{
nb -= iov[offset].MPID_IOV_LEN;
offset++;
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"partial write, request enqueued at head");
update_request(sreq, iov, n_iov, offset, nb);
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(vcch->conn->sock,
sreq->dev.iov + offset,
sreq->dev.iov_count - offset, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
break;
}
}
if (offset == n_iov)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"write complete, calling OnDataAvail fcn");
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
}
else {
int complete;
mpi_errno = reqFn( vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (!complete) {
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(
vcch->conn->sock, sreq->dev.iov,
sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
}
}
/* --BEGIN ERROR HANDLING-- */
else if (MPIR_ERR_GET_CLASS(rc) == MPIDU_SOCK_ERR_NOMEM)
{
MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,
"MPIDU_Sock_writev failed, out of memory");
sreq->status.MPI_ERROR = MPIR_ERR_MEMALLOCFAILED;
}
else
{
MPIU_DBG_MSG_D(CH3_CHANNEL,TYPICAL,
"MPIDU_Sock_writev failed, rc=%d", rc);
/* Connection just failed. Mark the request complete and
return an error. */
MPIU_DBG_VCCHSTATECHANGE(vc,VC_STATE_FAILED);
/* FIXME: Shouldn't the vc->state also change? */
vcch->state = MPIDI_CH3I_VC_STATE_FAILED;
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT - CH3U_Request_complete performs write barrier */
MPIDI_CH3U_Request_complete(sreq);
/* Return error to calling routine */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"send queue not empty, enqueuing");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING)
{
/* queuing the data so it can be sent later. */
MPIU_DBG_VCUSE(vc,"connecting. Enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED)
{
/* Form a new connection, queuing the data so it can be sent later. */
MPIU_DBG_VCUSE(vc,"unconnected. Enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
mpi_errno = MPIDI_CH3I_VC_post_connect(vc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPIU_DBG_VCUSE(vc,"still connecting. enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
MPIU_DBG_VCUSE(vc,"connection failed");
/* Connection failed. Mark the request complete and return an error. */
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete performs write barrier */
MPIDI_CH3U_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_ISENDV);
return mpi_errno;
}
