static inline int check_req_complete(MPIDI_VC_t *vc, MPID_Request *req, int *complete)
{
int mpi_errno = MPI_SUCCESS;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
reqFn = req->dev.OnDataAvail;
if (reqFn) {
*complete = 0;
/* XXX DJG FIXME this feels like a hack */
req->dev.iov_count = MPL_IOV_LIMIT;
req->dev.iov_offset = 0;
mpi_errno = reqFn(vc, req, complete);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
else {
*complete = 1;
mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
fn_fail:
return mpi_errno;
}
int MPIDI_CH3U_Handle_send_req(MPIDI_VC_t * vc, MPIR_Request * sreq, int *complete)
{
int mpi_errno = MPI_SUCCESS;
int (*reqFn) (MPIDI_VC_t *, MPIR_Request *, int *);
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
/* Use the associated function rather than switching on the old ca field */
/* Routines can call the attached function directly */
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIR_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
mpi_errno = MPID_Request_complete(sreq);
*complete = 1;
}
else {
mpi_errno = reqFn(vc, sreq, complete);
}
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
return mpi_errno;
fn_fail:
goto fn_exit;
}
void
MPID_nem_newmad_handle_sreq(MPID_Request *req)
{
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
#ifdef DEBUG
fprintf(stdout,"========> Completing Send req %p \n",req);
#endif
reqFn = req->dev.OnDataAvail;
if (!reqFn){
MPIDI_CH3U_Request_complete(req);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else{
MPIDI_VC_t *vc = req->ch.vc;
int complete = 0;
reqFn(vc, req, &complete);
if(!complete)
{
MPIU_Assert(complete == TRUE);
}
}
if (REQ_FIELD(req,iov) != NULL)
MPIU_Free((REQ_FIELD(req,iov)));
mpid_nem_newmad_pending_send_req--;
}
int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq,
int * complete)
{
static int in_routine ATTRIBUTE((unused)) = FALSE;
int mpi_errno = MPI_SUCCESS;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
MPIU_Assert(in_routine == FALSE);
in_routine = TRUE;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
MPIDI_CH3U_Request_complete(rreq);
*complete = TRUE;
}
else {
mpi_errno = reqFn( vc, rreq, complete );
}
in_routine = FALSE;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
return mpi_errno;
}
static inline int check_req_complete(MPIDI_VC_t *vc, MPID_Request *req, int *complete)
{
int mpi_errno = MPI_SUCCESS;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
reqFn = req->dev.OnDataAvail;
if (reqFn) {
*complete = 0;
mpi_errno = reqFn(vc, req, complete);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else {
*complete = 1;
MPIDI_CH3U_Request_complete(req);
}
fn_fail:
return mpi_errno;
}
int MPID_nem_lmt_dma_done_send(MPIDI_VC_t *vc, MPID_Request *sreq)
{
int mpi_errno = MPI_SUCCESS;
int complete = 0;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_DMA_DONE_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_DMA_DONE_SEND);
/* free cookie from RTS packet */
MPIU_Free(sreq->ch.s_cookie);
/* We shouldn't ever need to handle the more IOVs case here. The DONE
message should only be sent when all of the data is truly transferred.
However in the interest of robustness, we'll start to handle it and
assert if it looks like we were supposed to send more data for some
reason. */
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (complete) {
/* request was completed by the OnDataAvail fn */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
else {
/* There is more data to send. */
MPIU_Assert(("should never be incomplete!", 0));
}
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_DMA_DONE_SEND);
fn_exit:
return MPI_SUCCESS;
fn_fail:
goto fn_exit;
}
static int _mxm_handle_sreq(MPID_Request * req)
{
int complete = FALSE;
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
MPID_nem_mxm_vc_area *vc_area = NULL;
MPID_nem_mxm_req_area *req_area = NULL;
vc_area = VC_BASE(req->ch.vc);
req_area = REQ_BASE(req);
_dbg_mxm_out_buf(req_area->iov_buf[0].ptr,
(req_area->iov_buf[0].length >
16 ? 16 : req_area->iov_buf[0].length));
vc_area->pending_sends -= 1;
if (((req->dev.datatype_ptr != NULL) && (req->dev.tmpbuf != NULL))) {
MPIU_Free(req->dev.tmpbuf);
}
if (req_area->iov_count > MXM_MPICH_MAX_IOV) {
MPIU_Free(req_area->iov_buf);
req_area->iov_buf = req_area->tmp_buf;
req_area->iov_count = 0;
}
reqFn = req->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(req);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
MPIDI_VC_t *vc = req->ch.vc;
reqFn(vc, req, &complete);
if (!complete) {
MPIU_Assert(complete == TRUE);
}
}
return complete;
}
static int ReadMoreData( MPIDI_CH3I_Connection_t * conn, MPID_Request *rreq )
{
int mpi_errno = MPI_SUCCESS;
while (1) {
MPID_IOV * iovp;
MPIU_Size_t nb;
iovp = rreq->dev.iov;
mpi_errno = MPIDU_Sock_readv(conn->sock, iovp,
rreq->dev.iov_count, &nb);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|immedread", "ch3|sock|immedread %p %p %p",
rreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"immediate readv, vc=%p nb=" MPIDI_MSG_SZ_FMT ", rreq=0x%08x",
conn->vc, nb, rreq->handle));
if (nb > 0 && adjust_iov(&iovp, &rreq->dev.iov_count, nb)) {
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(rreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, rreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete) {
conn->recv_active = NULL; /* -- already set to NULL */
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
break;
}
}
else {
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting readv, vc=%p, rreq=0x%08x",
conn->vc, rreq->handle));
conn->recv_active = rreq;
mpi_errno = MPIDU_Sock_post_readv(conn->sock, iovp, rreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postread",
"ch3|sock|postread %p %p %p", rreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
break;
}
}
fn_fail:
return mpi_errno;
}
static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,"Socket event of type %d", event->op_type );
switch (event->op_type)
{
case MPIDU_SOCK_OP_READ:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
MPID_Request * rreq = conn->recv_active;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS)
{
/* FIXME: the following should be handled by the close
protocol */
if (MPIR_ERR_GET_CLASS(event->error) != MPIDU_SOCK_ERR_CONN_CLOSED) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
break;
}
/* --END ERROR HANDLING-- */
if (conn->state == CONN_STATE_CONNECTED)
{
if (conn->recv_active == NULL)
{
MPIDI_msg_sz_t buflen = sizeof (MPIDI_CH3_Pkt_t);
MPIU_Assert(conn->pkt.type < MPIDI_CH3_PKT_END_CH3);
mpi_errno = pktArray[conn->pkt.type]( conn->vc, &conn->pkt,
&buflen, &rreq );
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
MPIU_Assert(buflen == sizeof (MPIDI_CH3_Pkt_t));
if (rreq == NULL)
{
if (conn->state != CONN_STATE_CLOSING)
{
/* conn->recv_active = NULL; --
already set to NULL */
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
}
else
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
else /* incoming data */
{
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(rreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, rreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
conn->recv_active = NULL;
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to be read */
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
}
else if (conn->state == CONN_STATE_OPEN_LRECV_DATA)
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connopen_event( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
else /* Handling some internal connection establishment or
tear down packet */
{
mpi_errno = MPIDI_CH3_Sockconn_handle_conn_event( conn );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
break;
}
/* END OF SOCK_OP_READ */
case MPIDU_SOCK_OP_WRITE:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
/* --END ERROR HANDLING-- */
if (conn->send_active)
{
MPID_Request * sreq = conn->send_active;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to send */
{
for(;;)
{
MPID_IOV * iovp;
MPIU_Size_t nb;
iovp = sreq->dev.iov;
mpi_errno = MPIDU_Sock_writev(conn->sock, iovp, sreq->dev.iov_count, &nb);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|immedwrite", "ch3|sock|immedwrite %p %p %p",
sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"immediate writev, vc=%p, sreq=0x%08x, nb=" MPIDI_MSG_SZ_FMT,
conn->vc, sreq->handle, nb));
if (nb > 0 && adjust_iov(&iovp, &sreq->dev.iov_count, nb))
{
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
break;
}
}
else
{
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=%p, conn=%p, sreq=0x%08x",
conn->vc, conn, sreq->handle));
mpi_errno = MPIDU_Sock_post_writev(conn->sock, iovp, sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postwrite",
"ch3|sock|postwrite %p %p %p", sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
break;
}
}
}
}
else /* finished writing internal packet header */
{
/* the connection is not active yet */
mpi_errno = MPIDI_CH3_Sockconn_handle_connwrite( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
break;
}
/* END OF SOCK_OP_WRITE */
case MPIDU_SOCK_OP_ACCEPT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_accept_event();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CONNECT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connect_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr,
event->error );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CLOSE:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_close_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_WAKEUP:
{
MPIDI_CH3_Progress_signal_completion();
/* MPIDI_CH3I_progress_completion_count++; */
break;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_iSend(MPIDI_VC_t * vc, MPIR_Request * sreq, void * hdr,
intptr_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
int (*reqFn)(MPIDI_VC_t *, MPIR_Request *, int *);
MPIDI_CH3I_VC *vcch = &vc->ch;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_ISEND);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_ISEND);
MPIR_Assert( hdr_sz <= sizeof(MPIDI_CH3_Pkt_t) );
/* The sock channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
hdr_sz = sizeof(MPIDI_CH3_Pkt_t);
MPL_DBG_STMT(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t*)hdr));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
size_t nb;
int rc;
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPL_DBG_PKT(vcch->conn,hdr,"isend");
/* MT: need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
also try to write */
rc = MPIDI_CH3I_Sock_write(vcch->sock, hdr, hdr_sz, &nb);
if (rc == MPI_SUCCESS)
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
if (nb == hdr_sz)
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"write complete %" PRIdPTR " bytes, calling OnDataAvail fcn", nb);
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIR_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else {
int complete;
mpi_errno = reqFn( vc, sreq, &complete );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (!complete) {
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
(MPL_DBG_FDEST,
"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDI_CH3I_Sock_post_writev(
vcch->conn->sock, sreq->dev.iov,
sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
}
else
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"partial write of %" PRIdPTR " bytes, request enqueued at head", nb);
update_request(sreq, hdr, hdr_sz, nb);
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
(MPL_DBG_FDEST,"posting write, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDI_CH3I_Sock_post_write(vcch->conn->sock,
sreq->dev.iov[0].MPL_IOV_BUF,
sreq->dev.iov[0].MPL_IOV_LEN,
sreq->dev.iov[0].MPL_IOV_LEN, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
/* --BEGIN ERROR HANDLING-- */
else if (MPIR_ERR_GET_CLASS(rc) == MPIDI_CH3I_SOCK_ERR_NOMEM)
{
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,TYPICAL,
"MPIDI_CH3I_Sock_write failed, out of memory");
sreq->status.MPI_ERROR = MPIR_ERR_MEMALLOCFAILED;
}
else
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,TYPICAL,
"MPIDI_CH3I_Sock_write failed, rc=%d", rc);
/* Connection just failed. Mark the request complete and
return an error. */
MPL_DBG_VCCHSTATECHANGE(vc,VC_STATE_FAILED);
/* FIXME: Shouldn't the vc->state also change? */
vcch->state = MPIDI_CH3I_VC_STATE_FAILED;
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT -CH3U_Request_complete() performs write barrier */
MPID_Request_complete(sreq);
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,VERBOSE,"send queue not empty, enqueuing");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING) /* MT */
{
/* queuing the data so it can be sent later. */
MPL_DBG_VCUSE(vc,"connecting. enqueuing request");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED) /* MT */
{
/* Form a new connection, queuing the data so it can be sent later. */
MPL_DBG_VCUSE(vc,"unconnected. enqueuing request");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
mpi_errno = MPIDI_CH3I_VC_post_connect(vc);
if (mpi_errno) {
MPIR_ERR_POP(mpi_errno);
}
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPL_DBG_VCUSE(vc,"still connecting. Enqueuing request");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
/* Connection failed. Mark the request complete and return an error. */
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete() performs write barrier */
MPID_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
fn_fail:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_ISEND);
return mpi_errno;
}
int MPID_nem_scif_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header,
MPIDI_msg_sz_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
int iov_n;
MPID_IOV iov[MPID_IOV_LIMIT];
MPID_IOV *iov_p;
MPIDI_msg_sz_t offset = 0;
int complete;
uint64_t seqno = 0;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_SendNoncontig");
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
iov[0].MPID_IOV_BUF = header;
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
iov_n = MPID_IOV_LIMIT - 1;
mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &iov[1], &iov_n);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**loadsendiov");
iov_n += 1;
offset = 0;
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
offset =
MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend, iov, iov_n, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "scif_send noncontig " MPIDI_MSG_SZ_FMT, offset);
}
if (offset < iov[0].MPID_IOV_LEN) {
/* header was not yet sent, save it in req */
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) header;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & sreq->dev.pending_pkt;
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
}
/* check if whole iov was sent, and save any unsent portion of
* iov */
sreq->dev.iov_count = 0;
complete = 1;
for (iov_p = &iov[0]; iov_p < &iov[iov_n]; ++iov_p) {
if (offset < iov_p->MPID_IOV_LEN) {
sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_BUF =
(MPID_IOV_BUF_CAST) ((char *) iov_p->MPID_IOV_BUF + offset);
sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_LEN = iov_p->MPID_IOV_LEN - offset;
offset = 0;
++sreq->dev.iov_count;
complete = 0;
seqno = 0;
}
else
offset -= iov_p->MPID_IOV_LEN;
}
if (seqno)
complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);
if (complete) {
/* sent whole iov */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
seqno = 0;
}
/* enqueue request */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");
MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0));
RQ_SCIF(sreq)->seqno = seqno;
sreq->ch.vc = vc;
sreq->dev.iov_offset = 0;
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
/* this will be the first send on the queue: queue it and set
* the write flag on the pollfd */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
}
else {
/* there are other sends in the queue before this one: try to
* send from the queue */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
return mpi_errno;
fn_fail:
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
goto fn_exit;
}
int MPID_nem_scif_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t offset = 0;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
scifconn_t *sc = vc_scif->sc;
uint64_t seqno = 0;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iSendContig");
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
MPID_nem_scif_poll_send(sc->fd, &sc->csend)) {
MPID_IOV iov[2];
iov[0].MPID_IOV_BUF = hdr;
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
iov[1].MPID_IOV_BUF = data;
iov[1].MPID_IOV_LEN = data_sz;
offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE,
(MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d",
offset, sc->fd));
if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) {
/* sent whole message */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
if (seqno)
goto enqueue_request;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
else {
int complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
/* not completed: more to send */
goto enqueue_request;
}
}
}
/* save iov */
if (offset < sizeof(MPIDI_CH3_Pkt_t)) {
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr;
sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset;
sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset;
if (data_sz) {
sreq->dev.iov[1].MPID_IOV_BUF = data;
sreq->dev.iov[1].MPID_IOV_LEN = data_sz;
sreq->dev.iov_count = 2;
}
else
sreq->dev.iov_count = 1;
seqno = 0;
}
else {
sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t));
sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t));
sreq->dev.iov_count = 1;
seqno = 0;
}
enqueue_request:
/* enqueue request */
RQ_SCIF(sreq)->seqno = seqno;
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");
MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0));
sreq->ch.vc = vc;
sreq->dev.iov_offset = 0;
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
/* this will be the first send on the queue: queue it and set
* the write flag on the pollfd */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
}
else {
/* there are other sends in the queue before this one: try to
* send from the queue */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_scif_send_queued(MPIDI_VC_t * vc, reqq_t * send_queue)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq;
MPIDI_msg_sz_t offset;
MPID_IOV *iov;
int complete;
uint64_t seqno;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
MPIU_Assert(vc != NULL);
if (MPIDI_CH3I_Sendq_empty(*send_queue)) {
if (vc_scif->terminate)
MPID_nem_scif_vc_terminated(vc);
goto fn_exit;
}
while (!MPIDI_CH3I_Sendq_empty(*send_queue)) {
sreq = MPIDI_CH3I_Sendq_head(*send_queue);
seqno = RQ_SCIF(sreq)->seqno;
complete = 0;
if (seqno) {
/* We were just waiting for DMA to finish */
complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);
}
else if (MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "Sending %p", sreq);
iov = &sreq->dev.iov[sreq->dev.iov_offset];
offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend,
iov, sreq->dev.iov_count, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write " MPIDI_MSG_SZ_FMT, offset);
RQ_SCIF(sreq)->seqno = seqno;
complete = 1;
for (iov = &sreq->dev.iov[sreq->dev.iov_offset];
iov < &sreq->dev.iov[sreq->dev.iov_offset + sreq->dev.iov_count]; ++iov) {
if (offset < iov->MPID_IOV_LEN) {
iov->MPID_IOV_BUF = (char *) iov->MPID_IOV_BUF + offset;
iov->MPID_IOV_LEN -= offset;
/* iov_count should be equal to the number of iov's remaining */
sreq->dev.iov_count -= ((iov - sreq->dev.iov) - sreq->dev.iov_offset);
sreq->dev.iov_offset = iov - sreq->dev.iov;
complete = 0;
seqno = 0;
break;
}
offset -= iov->MPID_IOV_LEN;
}
if (seqno)
complete = 0;
}
if (!complete) {
/* writev couldn't write the entire iov, give up for now */
break;
}
else {
/* sent whole message */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
RQ_SCIF(sreq)->seqno = 0;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
}
complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
}
sreq->dev.iov_offset = 0;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
MPID_IOV * iov, int n_iov)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vcch = &vc->ch;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_ISENDV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_ISENDV);
MPIU_Assert(n_iov <= MPID_IOV_LIMIT);
MPIU_Assert(iov[0].MPID_IOV_LEN <= sizeof(MPIDI_CH3_Pkt_t));
/* The sock channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
MPIU_DBG_STMT(CH3_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)iov[0].MPID_IOV_BUF));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
MPIU_Size_t nb;
int rc;
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPIU_DBG_PKT(vcch->conn,(MPIDI_CH3_Pkt_t*)iov[0].MPID_IOV_BUF,
"isendv");
/* MT - need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
also try to write */
/* FIXME: the current code only agressively writes the first IOV.
Eventually it should be changed to agressively write
as much as possible. Ideally, the code would be shared between
the send routines and the progress engine. */
rc = MPIDU_Sock_writev(vcch->sock, iov, n_iov, &nb);
if (rc == MPI_SUCCESS)
{
int offset = 0;
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
while (offset < n_iov)
{
if ((int)iov[offset].MPID_IOV_LEN <= nb)
{
nb -= iov[offset].MPID_IOV_LEN;
offset++;
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"partial write, request enqueued at head");
update_request(sreq, iov, n_iov, offset, nb);
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(vcch->conn->sock,
sreq->dev.iov + offset,
sreq->dev.iov_count - offset, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
break;
}
}
if (offset == n_iov)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"write complete, calling OnDataAvail fcn");
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
}
else {
int complete;
mpi_errno = reqFn( vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (!complete) {
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(
vcch->conn->sock, sreq->dev.iov,
sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
}
}
/* --BEGIN ERROR HANDLING-- */
else if (MPIR_ERR_GET_CLASS(rc) == MPIDU_SOCK_ERR_NOMEM)
{
MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,
"MPIDU_Sock_writev failed, out of memory");
sreq->status.MPI_ERROR = MPIR_ERR_MEMALLOCFAILED;
}
else
{
MPIU_DBG_MSG_D(CH3_CHANNEL,TYPICAL,
"MPIDU_Sock_writev failed, rc=%d", rc);
/* Connection just failed. Mark the request complete and
return an error. */
MPIU_DBG_VCCHSTATECHANGE(vc,VC_STATE_FAILED);
/* FIXME: Shouldn't the vc->state also change? */
vcch->state = MPIDI_CH3I_VC_STATE_FAILED;
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT - CH3U_Request_complete performs write barrier */
MPIDI_CH3U_Request_complete(sreq);
/* Return error to calling routine */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"send queue not empty, enqueuing");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING)
{
/* queuing the data so it can be sent later. */
MPIU_DBG_VCUSE(vc,"connecting. Enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED)
{
/* Form a new connection, queuing the data so it can be sent later. */
MPIU_DBG_VCUSE(vc,"unconnected. Enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
mpi_errno = MPIDI_CH3I_VC_post_connect(vc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPIU_DBG_VCUSE(vc,"still connecting. enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
MPIU_DBG_VCUSE(vc,"connection failed");
/* Connection failed. Mark the request complete and return an error. */
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete performs write barrier */
MPIDI_CH3U_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_ISENDV);
return mpi_errno;
}
int MPID_nem_ofi_poll(int in_blocking_poll)
{
int complete = 0, mpi_errno = MPI_SUCCESS;
ssize_t ret;
cq_tagged_entry_t wc;
cq_err_entry_t error;
MPIDI_VC_t *vc;
MPID_Request *req;
req_fn reqFn;
BEGIN_FUNC(FCNAME);
do {
/* ----------------------------------------------------- */
/* Poll the completion queue */
/* The strategy here is */
/* ret>0 successfull poll, events returned */
/* ret==0 empty poll, no events/no error */
/* ret<0, error, but some error instances should not */
/* cause MPI to terminate */
/* ----------------------------------------------------- */
ret = fi_cq_read(gl_data.cq, /* Tagged completion queue */
(void *) &wc, /* OUT: Tagged completion entry */
1); /* Number of entries to poll */
if (ret > 0) {
if (NULL != wc.op_context) {
req = context_to_req(wc.op_context);
if (REQ_OFI(req)->event_callback) {
MPIDI_CH3I_NM_OFI_RC(REQ_OFI(req)->event_callback(&wc, req));
continue;
}
reqFn = req->dev.OnDataAvail;
if (reqFn) {
if (REQ_OFI(req)->pack_buffer) {
MPIU_Free(REQ_OFI(req)->pack_buffer);
}
vc = REQ_OFI(req)->vc;
complete = 0;
MPIDI_CH3I_NM_OFI_RC(reqFn(vc, req, &complete));
continue;
}
else {
MPIU_Assert(0);
}
}
else {
MPIU_Assert(0);
}
}
else if (ret == -FI_EAGAIN)
;
else if (ret < 0) {
if (ret == -FI_EAVAIL) {
ret = fi_cq_readerr(gl_data.cq, (void *) &error, 0);
if (error.err == FI_ETRUNC) {
/* ----------------------------------------------------- */
/* This error message should only be delivered on send */
/* events. We want to ignore truncation errors */
/* on the sender side, but complete the request anyway */
/* Other kinds of requests, this is fatal. */
/* ----------------------------------------------------- */
req = context_to_req(error.op_context);
if (req->kind == MPID_REQUEST_SEND) {
mpi_errno = REQ_OFI(req)->event_callback(NULL, req);
}
else if (req->kind == MPID_REQUEST_RECV) {
mpi_errno = REQ_OFI(req)->event_callback(&wc, req);
req->status.MPI_ERROR = MPI_ERR_TRUNCATE;
req->status.MPI_TAG = error.tag;
}
else {
mpi_errno = MPI_ERR_OTHER;
}
}
else if (error.err == FI_ECANCELED) {
req = context_to_req(error.op_context);
MPIR_STATUS_SET_CANCEL_BIT(req->status, TRUE);
}
else {
mpi_errno = MPI_ERR_OTHER;
}
}
else {
MPIR_ERR_CHKANDJUMP4(1, mpi_errno, MPI_ERR_OTHER, "**ofi_poll",
"**ofi_poll %s %d %s %s", __SHORT_FILE__,
__LINE__, FCNAME, fi_strerror(-ret));
}
}
} while (in_blocking_poll && (ret > 0));
END_FUNC_RC(FCNAME);
}