int MPIDI_CH3U_Handle_recv_req(MPIDI_VC_t * vc, MPID_Request * rreq,
int * complete)
{
static int in_routine ATTRIBUTE((unused)) = FALSE;
int mpi_errno = MPI_SUCCESS;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
MPIU_Assert(in_routine == FALSE);
in_routine = TRUE;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
MPIDI_CH3U_Request_complete(rreq);
*complete = TRUE;
}
else {
mpi_errno = reqFn( vc, rreq, complete );
}
in_routine = FALSE;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_RECV_REQ);
return mpi_errno;
}
int MPIDI_CH3U_Handle_send_req(MPIDI_VC_t * vc, MPIR_Request * sreq, int *complete)
{
int mpi_errno = MPI_SUCCESS;
int (*reqFn) (MPIDI_VC_t *, MPIR_Request *, int *);
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
/* Use the associated function rather than switching on the old ca field */
/* Routines can call the attached function directly */
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIR_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
mpi_errno = MPID_Request_complete(sreq);
*complete = 1;
}
else {
mpi_errno = reqFn(vc, sreq, complete);
}
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_SEND_REQ);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_newmad_anysource_matched(MPID_Request *rreq)
{
/* This function is called when an anysource request in the posted
receive queue is matched and dequeued */
nm_sr_request_t *nmad_request = NULL;
int ret;
int matched = FALSE;
#ifdef DEBUG
fprintf(stdout,"========> Any Source : MPID_nem_newmad_anysource_matched , req is %p\n",rreq);
#endif
MPID_NEM_NMAD_GET_REQ_FROM_HASH(rreq,nmad_request);
if(nmad_request != NULL)
{
#ifdef DEBUG
fprintf(stdout,"========> Any Source nmad req found :%p \n",nmad_request);
#endif
ret = nm_sr_rcancel(mpid_nem_newmad_session,nmad_request);
if (ret != NM_ESUCCESS)
{
#ifdef DEBUG
fprintf(stdout,"========> Any Source nmad req (%p) not cancelled \n",nmad_request);
#endif
size_t size;
nm_tag_t match_info;
MPIU_Assert(MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_RESP);
ret = nm_sr_rwait(mpid_nem_newmad_session,nmad_request);
MPIU_Assert(ret == NM_ESUCCESS);
nm_sr_request_unset_completion_queue(mpid_nem_newmad_session,nmad_request);
nm_sr_get_rtag(mpid_nem_newmad_session,nmad_request,&match_info);
nm_sr_get_size(mpid_nem_newmad_session,nmad_request,&size);
MPID_nem_newmad_handle_rreq(rreq,match_info, size);
matched = TRUE;
}
else
{
MPID_Segment_free(rreq->dev.segment_ptr);
if (REQ_FIELD(rreq,iov) != NULL)
MPIU_Free(REQ_FIELD(rreq,iov));
}
MPIU_Free(nmad_request);
}
return matched;
}
void MPIDI_CH3U_Buffer_free( MPID_Request * request_ptr )
{
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_BUFFER_FREE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_BUFFER_FREE);
if ( ( MPIDI_REQUEST_TYPE_SEND == MPIDI_Request_get_type(request_ptr) ) &&
( MPIDI_Request_get_self_zerocopy_flag(request_ptr) ) &&
( !(FG_is_within_same_HWP(request_ptr->dev.match.parts.dest_rank, request_ptr->comm, NULL)) ) ) {
/* MPI_Izsend<=>MPI_Wait/Test (and variants) pairing: Freeing buf_handle */
MPIU_Assert( (request_ptr->dev.user_buf_handle) && *(request_ptr->dev.user_buf_handle) );
MPIU_Free(*(request_ptr->dev.user_buf_handle));
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_BUFFER_FREE);
}
/* XXX DJG FIXME at some point this should poll, much like the newtcp module.
But then we have that whole pollfd array to manage, which we don't really
need until this proof-of-concept proves itself. */
int MPID_nem_lmt_vmsplice_progress(void)
{
int mpi_errno = MPI_SUCCESS;
struct lmt_vmsplice_node *prev = NULL;
struct lmt_vmsplice_node *free_me = NULL;
struct lmt_vmsplice_node *cur = outstanding_head;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_VMSPLICE_PROGRESS);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_VMSPLICE_PROGRESS);
while (cur) {
int complete = 0;
switch (MPIDI_Request_get_type(cur->req)) {
case MPIDI_REQUEST_TYPE_RECV:
mpi_errno = do_readv(cur->req, cur->pipe_fd, cur->req->dev.iov,
&cur->req->dev.iov_offset,
&cur->req->dev.iov_count, &complete);
/* FIXME: set the error status of the req and complete it, rather than POP */
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
break;
case MPIDI_REQUEST_TYPE_SEND:
mpi_errno = do_vmsplice(cur->req, cur->pipe_fd, cur->req->dev.iov,
&cur->req->dev.iov_offset,
&cur->req->dev.iov_count, &complete);
/* FIXME: set the error status of the req and complete it, rather than POP */
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
break;
default:
MPIR_ERR_INTERNALANDJUMP(mpi_errno, "unexpected request type");
break;
}
if (complete) {
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete");
/* remove the node from the list */
if (cur == outstanding_head) {
outstanding_head = cur->next;
prev = NULL;
free_me = cur;
cur = cur->next;
}
else {
prev->next = cur->next;
prev = cur;
free_me = cur;
cur = cur->next;
}
if (free_me) MPL_free(free_me);
--MPID_nem_local_lmt_pending;
}
if (!cur) break; /* we might have made cur NULL above */
prev = cur;
cur = cur->next;
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_VMSPLICE_PROGRESS);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* MPIDI_CH3I_SendNoncontig - Sends a message by packing
directly into cells. The caller must initialize sreq->dev.segment
as well as segment_first and segment_size. */
int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz )
{
int mpi_errno = MPI_SUCCESS;
int again = 0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header);
MPIU_THREAD_CS_ENTER(MPIDCOMM,);
if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */
{
/* send queue is not empty, enqueue the request then check to
see if we can send any now */
MPIDI_DBG_PRINTF((55, FCNAME, "enqueuing"));
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
sreq->ch.noncontig = TRUE;
sreq->ch.header_sz = hdr_sz;
sreq->ch.vc = vc;
MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq);
mpi_errno = MPIDI_CH3I_Shm_send_progress();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
goto fn_exit;
}
/* send as many cells of data as you can */
MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, header, hdr_sz, vc, &again);
while(!again && sreq->dev.segment_first < sreq->dev.segment_size)
MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again);
if (again)
{
/* we didn't finish sending everything */
sreq->ch.noncontig = TRUE;
sreq->ch.vc = vc;
if (sreq->dev.segment_first == 0) /* nothing was sent, save header */
{
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
sreq->ch.header_sz = hdr_sz;
}
else
{
/* part of message was sent, make this req an active send */
MPIU_Assert(MPIDI_CH3I_shm_active_send == NULL);
MPIDI_CH3I_shm_active_send = sreq;
}
MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq);
goto fn_exit;
}
/* finished sending all data, complete the request */
if (!sreq->dev.OnDataAvail)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
else
{
int complete = 0;
mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_Assert(complete); /* all data has been sent, we should always complete */
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
fn_exit:
MPIU_THREAD_CS_EXIT(MPIDCOMM,);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3U_Request_load_recv_iov(MPID_Request * const rreq)
{
MPI_Aint last;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_RECV_IOV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_RECV_IOV);
if (rreq->dev.segment_first < rreq->dev.segment_size)
{
/* still reading data that needs to go into the user buffer */
if (MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_ACCUM_RECV &&
MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_ACCUM_RECV &&
MPIDI_Request_get_srbuf_flag(rreq))
{
MPIDI_msg_sz_t data_sz;
MPIDI_msg_sz_t tmpbuf_sz;
/* Once a SRBuf is in use, we continue to use it since a small
amount of data may already be present at the beginning
of the buffer. This data is left over from the previous unpack,
most like a result of alignment issues. NOTE: we
could force the use of the SRBuf only
when (rreq->dev.tmpbuf_off > 0)... */
data_sz = rreq->dev.segment_size - rreq->dev.segment_first -
rreq->dev.tmpbuf_off;
MPIU_Assert(data_sz > 0);
tmpbuf_sz = rreq->dev.tmpbuf_sz - rreq->dev.tmpbuf_off;
if (data_sz > tmpbuf_sz)
{
data_sz = tmpbuf_sz;
}
rreq->dev.iov[0].MPID_IOV_BUF =
(MPID_IOV_BUF_CAST)((char *) rreq->dev.tmpbuf +
rreq->dev.tmpbuf_off);
rreq->dev.iov[0].MPID_IOV_LEN = data_sz;
rreq->dev.iov_offset = 0;
rreq->dev.iov_count = 1;
MPIU_Assert(rreq->dev.segment_first + data_sz +
rreq->dev.tmpbuf_off <= rreq->dev.recv_data_sz);
if (rreq->dev.segment_first + data_sz + rreq->dev.tmpbuf_off ==
rreq->dev.recv_data_sz)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"updating rreq to read the remaining data into the SRBuf");
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackSRBufComplete;
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"updating rreq to read more data into the SRBuf");
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackSRBufReloadIOV;
}
goto fn_exit;
}
last = rreq->dev.segment_size;
rreq->dev.iov_count = MPID_IOV_LIMIT;
rreq->dev.iov_offset = 0;
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
"pre-upv: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT ", iov_n=%d",
rreq->dev.segment_first, last, rreq->dev.iov_count));
MPIU_Assert(rreq->dev.segment_first < last);
MPIU_Assert(last > 0);
MPID_Segment_unpack_vector(rreq->dev.segment_ptr,
rreq->dev.segment_first,
&last, &rreq->dev.iov[0], &rreq->dev.iov_count);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
"post-upv: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT ", iov_n=%d, iov_offset=%lld",
rreq->dev.segment_first, last, rreq->dev.iov_count, (long long)rreq->dev.iov_offset));
MPIU_Assert(rreq->dev.iov_count >= 0 && rreq->dev.iov_count <=
MPID_IOV_LIMIT);
/* --BEGIN ERROR HANDLING-- */
if (rreq->dev.iov_count == 0)
{
/* If the data can't be unpacked, the we have a mis-match between
the datatype and the amount of data received. Adjust
the segment info so that the remaining data is received and
thrown away. */
rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TYPE,
"**dtypemismatch", 0);
MPIR_STATUS_SET_COUNT(rreq->status, rreq->dev.segment_first);
rreq->dev.segment_size = rreq->dev.segment_first;
mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
goto fn_exit;
}
else
{
MPIU_Assert(rreq->dev.iov_offset < rreq->dev.iov_count);
}
/* --END ERROR HANDLING-- */
if (last == rreq->dev.recv_data_sz)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"updating rreq to read the remaining data directly into the user buffer");
/* Eventually, use OnFinal for this instead */
rreq->dev.OnDataAvail = rreq->dev.OnFinal;
}
else if (MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_ACCUM_RECV ||
MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_GET_ACCUM_RECV ||
(last == rreq->dev.segment_size ||
(last - rreq->dev.segment_first) / rreq->dev.iov_count >= MPIDI_IOV_DENSITY_MIN))
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"updating rreq to read more data directly into the user buffer");
rreq->dev.segment_first = last;
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_ReloadIOV;
}
else
{
/* Too little data would have been received using an IOV.
We will start receiving data into a SRBuf and unpacking it
later. */
MPIU_Assert(MPIDI_Request_get_srbuf_flag(rreq) == FALSE);
MPIDI_CH3U_SRBuf_alloc(rreq,
rreq->dev.segment_size - rreq->dev.segment_first);
rreq->dev.tmpbuf_off = 0;
/* --BEGIN ERROR HANDLING-- */
if (rreq->dev.tmpbuf_sz == 0)
{
/* FIXME - we should drain the data off the pipe here, but we
don't have a buffer to drain it into. should this be
a fatal error? */
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"SRBuf allocation failure");
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem",
"**nomem %d",
rreq->dev.segment_size - rreq->dev.segment_first);
rreq->status.MPI_ERROR = mpi_errno;
goto fn_exit;
}
/* --END ERROR HANDLING-- */
/* fill in the IOV using a recursive call */
mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
}
}
else
{
/* receive and toss any extra data that does not fit in the user's
buffer */
MPIDI_msg_sz_t data_sz;
data_sz = rreq->dev.recv_data_sz - rreq->dev.segment_first;
if (!MPIDI_Request_get_srbuf_flag(rreq))
{
MPIDI_CH3U_SRBuf_alloc(rreq, data_sz);
/* --BEGIN ERROR HANDLING-- */
if (rreq->dev.tmpbuf_sz == 0)
{
MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"SRBuf allocation failure");
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0);
rreq->status.MPI_ERROR = mpi_errno;
goto fn_exit;
}
/* --END ERROR HANDLING-- */
}
if (data_sz <= rreq->dev.tmpbuf_sz)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"updating rreq to read overflow data into the SRBuf and complete");
rreq->dev.iov[0].MPID_IOV_LEN = data_sz;
MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
/* Eventually, use OnFinal for this instead */
rreq->dev.OnDataAvail = rreq->dev.OnFinal;
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"updating rreq to read overflow data into the SRBuf and reload IOV");
rreq->dev.iov[0].MPID_IOV_LEN = rreq->dev.tmpbuf_sz;
rreq->dev.segment_first += rreq->dev.tmpbuf_sz;
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_ReloadIOV;
}
rreq->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)rreq->dev.tmpbuf;
rreq->dev.iov_count = 1;
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_RECV_IOV);
return mpi_errno;
}
int
MPID_nem_newmad_poll(int in_blocking_poll)
{
nm_sr_request_t *p_request = NULL;
nm_tag_t match_info = 0;
int mpi_errno = MPI_SUCCESS;
nm_sr_send_success(mpid_nem_newmad_session, &p_request);
if (p_request != NULL)
{
MPID_nem_newmad_unified_req_t *ref;
MPID_Request *req;
MPID_Request_kind_t kind;
MPIR_Context_id_t ctxt;
nm_sr_get_stag(mpid_nem_newmad_session,p_request, &match_info);
NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
nm_sr_get_ref(mpid_nem_newmad_session,p_request,(void *)&ref);
req = &(ref->mpi_req);
MPIU_Assert(req != NULL);
kind = req->kind;
if(ctxt == NEM_NMAD_INTRA_CTXT)
{
if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
{
MPID_nem_newmad_handle_sreq(req);
}
}
else
{
if ((kind == MPID_REQUEST_SEND) || (kind == MPID_PREQUEST_SEND))
{
MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
MPID_nem_newmad_handle_sreq(req);
}
}
}
nm_sr_recv_success(mpid_nem_newmad_session, &p_request);
if (p_request != NULL)
{
MPID_nem_newmad_unified_req_t *ref;
MPID_Request *req;
MPID_Request_kind_t kind;
MPIR_Context_id_t ctxt;
size_t size;
nm_sr_get_ref(mpid_nem_newmad_session,p_request,(void *)&ref);
req = &(ref->mpi_req);
MPIU_Assert(req != NULL);
kind = req->kind;
nm_sr_get_size(mpid_nem_newmad_session, p_request, &size);
nm_sr_get_rtag(mpid_nem_newmad_session,p_request, &match_info);
NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
if(ctxt == NEM_NMAD_INTRA_CTXT)
{
MPID_nem_newmad_internal_req_t *adi_req = &(ref->nem_newmad_req);
if (kind == MPID_REQUEST_RECV)
{
if (size <= sizeof(MPIDI_CH3_PktGeneric_t))
{
MPID_nem_handle_pkt(adi_req->vc,(char *)&(adi_req->pending_pkt),(MPIDI_msg_sz_t)(size));
}
else
{
MPID_nem_handle_pkt(adi_req->vc,(char *)(adi_req->tmpbuf),(MPIDI_msg_sz_t)(adi_req->tmpbuf_sz));
MPIU_Free(adi_req->tmpbuf);
}
nm_core_disable_progression(mpid_nem_newmad_session->p_core);
MPID_nem_newmad_internal_req_enqueue(adi_req);
nm_core_enable_progression(mpid_nem_newmad_session->p_core);
}
else
{
MPIU_Assert(0);
}
}
else
{
if ((kind == MPID_REQUEST_RECV) || (kind == MPID_PREQUEST_RECV))
{
int found = FALSE;
nm_sr_request_t *nmad_request = NULL;
MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIU_THREAD_CS_ENTER(MSGQUEUE,req);
MPID_NEM_NMAD_GET_REQ_FROM_HASH(req,nmad_request);
if(nmad_request != NULL)
{
MPIU_Assert(req->dev.match.parts.rank == MPI_ANY_SOURCE);
MPIU_Free(nmad_request);
}
found = MPIDI_CH3U_Recvq_DP(req);
if(found){
MPID_nem_newmad_handle_rreq(req,match_info,size);
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,req);
}
else
{
fprintf(stdout, ">>>>>>>>>>>>> ERROR: Wrong req type : %i (%p)\n",(int)kind,req);
MPIU_Assert(0);
}
}
}
fn_exit:
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
static int ReadMoreData( MPIDI_CH3I_Connection_t * conn, MPID_Request *rreq )
{
int mpi_errno = MPI_SUCCESS;
while (1) {
MPID_IOV * iovp;
MPIU_Size_t nb;
iovp = rreq->dev.iov;
mpi_errno = MPIDU_Sock_readv(conn->sock, iovp,
rreq->dev.iov_count, &nb);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|immedread", "ch3|sock|immedread %p %p %p",
rreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"immediate readv, vc=%p nb=" MPIDI_MSG_SZ_FMT ", rreq=0x%08x",
conn->vc, nb, rreq->handle));
if (nb > 0 && adjust_iov(&iovp, &rreq->dev.iov_count, nb)) {
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(rreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, rreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete) {
conn->recv_active = NULL; /* -- already set to NULL */
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
break;
}
}
else {
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting readv, vc=%p, rreq=0x%08x",
conn->vc, rreq->handle));
conn->recv_active = rreq;
mpi_errno = MPIDU_Sock_post_readv(conn->sock, iovp, rreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postread",
"ch3|sock|postread %p %p %p", rreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
break;
}
}
fn_fail:
return mpi_errno;
}
static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,"Socket event of type %d", event->op_type );
switch (event->op_type)
{
case MPIDU_SOCK_OP_READ:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
MPID_Request * rreq = conn->recv_active;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS)
{
/* FIXME: the following should be handled by the close
protocol */
if (MPIR_ERR_GET_CLASS(event->error) != MPIDU_SOCK_ERR_CONN_CLOSED) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
break;
}
/* --END ERROR HANDLING-- */
if (conn->state == CONN_STATE_CONNECTED)
{
if (conn->recv_active == NULL)
{
MPIDI_msg_sz_t buflen = sizeof (MPIDI_CH3_Pkt_t);
MPIU_Assert(conn->pkt.type < MPIDI_CH3_PKT_END_CH3);
mpi_errno = pktArray[conn->pkt.type]( conn->vc, &conn->pkt,
&buflen, &rreq );
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
MPIU_Assert(buflen == sizeof (MPIDI_CH3_Pkt_t));
if (rreq == NULL)
{
if (conn->state != CONN_STATE_CLOSING)
{
/* conn->recv_active = NULL; --
already set to NULL */
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
}
else
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
else /* incoming data */
{
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(rreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, rreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
conn->recv_active = NULL;
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to be read */
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
}
else if (conn->state == CONN_STATE_OPEN_LRECV_DATA)
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connopen_event( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
else /* Handling some internal connection establishment or
tear down packet */
{
mpi_errno = MPIDI_CH3_Sockconn_handle_conn_event( conn );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
break;
}
/* END OF SOCK_OP_READ */
case MPIDU_SOCK_OP_WRITE:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
/* --END ERROR HANDLING-- */
if (conn->send_active)
{
MPID_Request * sreq = conn->send_active;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to send */
{
for(;;)
{
MPID_IOV * iovp;
MPIU_Size_t nb;
iovp = sreq->dev.iov;
mpi_errno = MPIDU_Sock_writev(conn->sock, iovp, sreq->dev.iov_count, &nb);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|immedwrite", "ch3|sock|immedwrite %p %p %p",
sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"immediate writev, vc=%p, sreq=0x%08x, nb=" MPIDI_MSG_SZ_FMT,
conn->vc, sreq->handle, nb));
if (nb > 0 && adjust_iov(&iovp, &sreq->dev.iov_count, nb))
{
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
break;
}
}
else
{
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=%p, conn=%p, sreq=0x%08x",
conn->vc, conn, sreq->handle));
mpi_errno = MPIDU_Sock_post_writev(conn->sock, iovp, sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postwrite",
"ch3|sock|postwrite %p %p %p", sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
break;
}
}
}
}
else /* finished writing internal packet header */
{
/* the connection is not active yet */
mpi_errno = MPIDI_CH3_Sockconn_handle_connwrite( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
break;
}
/* END OF SOCK_OP_WRITE */
case MPIDU_SOCK_OP_ACCEPT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_accept_event();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CONNECT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connect_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr,
event->error );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CLOSE:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_close_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_WAKEUP:
{
MPIDI_CH3_Progress_signal_completion();
/* MPIDI_CH3I_progress_completion_count++; */
break;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_iSend(MPIDI_VC_t * vc, MPIR_Request * sreq, void * hdr,
intptr_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
int (*reqFn)(MPIDI_VC_t *, MPIR_Request *, int *);
MPIDI_CH3I_VC *vcch = &vc->ch;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3_ISEND);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3_ISEND);
MPIR_Assert( hdr_sz <= sizeof(MPIDI_CH3_Pkt_t) );
/* The sock channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
hdr_sz = sizeof(MPIDI_CH3_Pkt_t);
MPL_DBG_STMT(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t*)hdr));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
size_t nb;
int rc;
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPL_DBG_PKT(vcch->conn,hdr,"isend");
/* MT: need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
also try to write */
rc = MPIDI_CH3I_Sock_write(vcch->sock, hdr, hdr_sz, &nb);
if (rc == MPI_SUCCESS)
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
if (nb == hdr_sz)
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"write complete %" PRIdPTR " bytes, calling OnDataAvail fcn", nb);
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIR_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
}
else {
int complete;
mpi_errno = reqFn( vc, sreq, &complete );
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (!complete) {
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
(MPL_DBG_FDEST,
"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDI_CH3I_Sock_post_writev(
vcch->conn->sock, sreq->dev.iov,
sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
}
else
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
"partial write of %" PRIdPTR " bytes, request enqueued at head", nb);
update_request(sreq, hdr, hdr_sz, nb);
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_CHANNEL,VERBOSE,
(MPL_DBG_FDEST,"posting write, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDI_CH3I_Sock_post_write(vcch->conn->sock,
sreq->dev.iov[0].MPL_IOV_BUF,
sreq->dev.iov[0].MPL_IOV_LEN,
sreq->dev.iov[0].MPL_IOV_LEN, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
/* --BEGIN ERROR HANDLING-- */
else if (MPIR_ERR_GET_CLASS(rc) == MPIDI_CH3I_SOCK_ERR_NOMEM)
{
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,TYPICAL,
"MPIDI_CH3I_Sock_write failed, out of memory");
sreq->status.MPI_ERROR = MPIR_ERR_MEMALLOCFAILED;
}
else
{
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL,TYPICAL,
"MPIDI_CH3I_Sock_write failed, rc=%d", rc);
/* Connection just failed. Mark the request complete and
return an error. */
MPL_DBG_VCCHSTATECHANGE(vc,VC_STATE_FAILED);
/* FIXME: Shouldn't the vc->state also change? */
vcch->state = MPIDI_CH3I_VC_STATE_FAILED;
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT -CH3U_Request_complete() performs write barrier */
MPID_Request_complete(sreq);
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL,VERBOSE,"send queue not empty, enqueuing");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING) /* MT */
{
/* queuing the data so it can be sent later. */
MPL_DBG_VCUSE(vc,"connecting. enqueuing request");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED) /* MT */
{
/* Form a new connection, queuing the data so it can be sent later. */
MPL_DBG_VCUSE(vc,"unconnected. enqueuing request");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
mpi_errno = MPIDI_CH3I_VC_post_connect(vc);
if (mpi_errno) {
MPIR_ERR_POP(mpi_errno);
}
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPL_DBG_VCUSE(vc,"still connecting. Enqueuing request");
update_request(sreq, hdr, hdr_sz, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
/* Connection failed. Mark the request complete and return an error. */
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete() performs write barrier */
MPID_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
fn_fail:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3_ISEND);
return mpi_errno;
}
int MPID_nem_scif_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr,
MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_msg_sz_t offset = 0;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
scifconn_t *sc = vc_scif->sc;
uint64_t seqno = 0;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iSendContig");
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr);
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
MPID_nem_scif_poll_send(sc->fd, &sc->csend)) {
MPID_IOV iov[2];
iov[0].MPID_IOV_BUF = hdr;
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
iov[1].MPID_IOV_BUF = data;
iov[1].MPID_IOV_LEN = data_sz;
offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE,
(MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d",
offset, sc->fd));
if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) {
/* sent whole message */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
if (seqno)
goto enqueue_request;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
else {
int complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
/* not completed: more to send */
goto enqueue_request;
}
}
}
/* save iov */
if (offset < sizeof(MPIDI_CH3_Pkt_t)) {
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr;
sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset;
sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset;
if (data_sz) {
sreq->dev.iov[1].MPID_IOV_BUF = data;
sreq->dev.iov[1].MPID_IOV_LEN = data_sz;
sreq->dev.iov_count = 2;
}
else
sreq->dev.iov_count = 1;
seqno = 0;
}
else {
sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t));
sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t));
sreq->dev.iov_count = 1;
seqno = 0;
}
enqueue_request:
/* enqueue request */
RQ_SCIF(sreq)->seqno = seqno;
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");
MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0));
sreq->ch.vc = vc;
sreq->dev.iov_offset = 0;
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
/* this will be the first send on the queue: queue it and set
* the write flag on the pollfd */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
}
else {
/* there are other sends in the queue before this one: try to
* send from the queue */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_scif_send_queued(MPIDI_VC_t * vc, reqq_t * send_queue)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq;
MPIDI_msg_sz_t offset;
MPID_IOV *iov;
int complete;
uint64_t seqno;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
MPIU_Assert(vc != NULL);
if (MPIDI_CH3I_Sendq_empty(*send_queue)) {
if (vc_scif->terminate)
MPID_nem_scif_vc_terminated(vc);
goto fn_exit;
}
while (!MPIDI_CH3I_Sendq_empty(*send_queue)) {
sreq = MPIDI_CH3I_Sendq_head(*send_queue);
seqno = RQ_SCIF(sreq)->seqno;
complete = 0;
if (seqno) {
/* We were just waiting for DMA to finish */
complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);
}
else if (MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "Sending %p", sreq);
iov = &sreq->dev.iov[sreq->dev.iov_offset];
offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend,
iov, sreq->dev.iov_count, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write " MPIDI_MSG_SZ_FMT, offset);
RQ_SCIF(sreq)->seqno = seqno;
complete = 1;
for (iov = &sreq->dev.iov[sreq->dev.iov_offset];
iov < &sreq->dev.iov[sreq->dev.iov_offset + sreq->dev.iov_count]; ++iov) {
if (offset < iov->MPID_IOV_LEN) {
iov->MPID_IOV_BUF = (char *) iov->MPID_IOV_BUF + offset;
iov->MPID_IOV_LEN -= offset;
/* iov_count should be equal to the number of iov's remaining */
sreq->dev.iov_count -= ((iov - sreq->dev.iov) - sreq->dev.iov_offset);
sreq->dev.iov_offset = iov - sreq->dev.iov;
complete = 0;
seqno = 0;
break;
}
offset -= iov->MPID_IOV_LEN;
}
if (seqno)
complete = 0;
}
if (!complete) {
/* writev couldn't write the entire iov, give up for now */
break;
}
else {
/* sent whole message */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
RQ_SCIF(sreq)->seqno = 0;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
}
complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
}
sreq->dev.iov_offset = 0;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* MPIDI_CH3I_SendNoncontig - Sends a message by packing
directly into cells. The caller must initialize sreq->dev.segment
as well as segment_first and segment_size. */
int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPIR_Request *sreq, void *header, intptr_t hdr_sz,
MPL_IOV *hdr_iov, int n_hdr_iov)
{
int mpi_errno = MPI_SUCCESS;
int again = 0;
intptr_t orig_segment_first = sreq->dev.segment_first;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header);
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
if (n_hdr_iov > 0) {
/* translate segments to iovs and combine with the extended header iov. */
mpi_errno = MPIDI_CH3_SendNoncontig_iov(vc, sreq, header, hdr_sz,
hdr_iov, n_hdr_iov);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
goto fn_exit;
}
if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */
{
/* send queue is not empty, enqueue the request then check to
see if we can send any now */
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER, TERSE, "enqueuing");
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
sreq->ch.noncontig = TRUE;
sreq->ch.header_sz = hdr_sz;
sreq->ch.vc = vc;
MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq);
mpi_errno = MPIDI_CH3I_Shm_send_progress();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
goto fn_exit;
}
/* send as many cells of data as you can */
MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size,
header, hdr_sz, vc, &again);
while(!again && sreq->dev.segment_first < sreq->dev.segment_size)
MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again);
if (again)
{
/* we didn't finish sending everything */
sreq->ch.noncontig = TRUE;
sreq->ch.vc = vc;
if (sreq->dev.segment_first == orig_segment_first) /* nothing was sent, save header */
{
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
sreq->ch.header_sz = hdr_sz;
}
else
{
/* part of message was sent, make this req an active send */
MPIR_Assert(MPIDI_CH3I_shm_active_send == NULL);
MPIDI_CH3I_shm_active_send = sreq;
}
MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq);
goto fn_exit;
}
/* finished sending all data, complete the request */
if (!sreq->dev.OnDataAvail)
{
MPIR_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
mpi_errno = MPID_Request_complete(sreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
else
{
int complete = 0;
mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIR_Assert(complete); /* all data has been sent, we should always complete */
MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
fn_exit:
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Startall(int count, MPID_Request * requests[])
{
int i;
int rc;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_STARTALL);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_STARTALL);
for (i = 0; i < count; i++)
{
MPID_Request * const preq = requests[i];
/* FIXME: The odd 7th arg (match.context_id - comm->context_id)
is probably to get the context offset. Do we really need the
context offset? Is there any case where the offset isn't zero? */
switch (MPIDI_Request_get_type(preq))
{
case MPIDI_REQUEST_TYPE_RECV:
{
rc = MPID_Irecv(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->recvcontext_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_SEND:
{
rc = MPID_Isend(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->context_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_RSEND:
{
rc = MPID_Irsend(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->context_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_SSEND:
{
rc = MPID_Issend(preq->dev.user_buf, preq->dev.user_count, preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm, preq->dev.match.parts.context_id - preq->comm->context_id,
&preq->partner_request);
break;
}
case MPIDI_REQUEST_TYPE_BSEND:
{
MPI_Request sreq_handle;
rc = MPIR_Ibsend_impl(preq->dev.user_buf, preq->dev.user_count,
preq->dev.datatype, preq->dev.match.parts.rank,
preq->dev.match.parts.tag, preq->comm,
&sreq_handle);
if (rc == MPI_SUCCESS)
{
MPID_Request_get_ptr(sreq_handle, preq->partner_request);
}
break;
}
default:
{
/* --BEGIN ERROR HANDLING-- */
rc = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_INTERN, "**ch3|badreqtype",
"**ch3|badreqtype %d", MPIDI_Request_get_type(preq));
/* --END ERROR HANDLING-- */
}
}
if (rc == MPI_SUCCESS)
{
preq->status.MPI_ERROR = MPI_SUCCESS;
preq->cc_ptr = &preq->partner_request->cc;
}
/* --BEGIN ERROR HANDLING-- */
else
{
/* If a failure occurs attempting to start the request, then we
assume that partner request was not created, and stuff
the error code in the persistent request. The wait and test
routines will look at the error code in the persistent
request if a partner request is not present. */
preq->partner_request = NULL;
preq->status.MPI_ERROR = rc;
preq->cc_ptr = &preq->cc;
MPID_cc_set(&preq->cc, 0);
}
/* --END ERROR HANDLING-- */
}
MPIDI_FUNC_EXIT(MPID_STATE_MPID_STARTALL);
return mpi_errno;
}
int MPIDI_CH3_iSendv(MPIDI_VC_t * vc, MPID_Request * sreq,
MPID_IOV * iov, int n_iov)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vcch = &vc->ch;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_ISENDV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_ISENDV);
MPIU_Assert(n_iov <= MPID_IOV_LIMIT);
MPIU_Assert(iov[0].MPID_IOV_LEN <= sizeof(MPIDI_CH3_Pkt_t));
/* The sock channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
MPIU_DBG_STMT(CH3_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)iov[0].MPID_IOV_BUF));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
MPIU_Size_t nb;
int rc;
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPIU_DBG_PKT(vcch->conn,(MPIDI_CH3_Pkt_t*)iov[0].MPID_IOV_BUF,
"isendv");
/* MT - need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
also try to write */
/* FIXME: the current code only agressively writes the first IOV.
Eventually it should be changed to agressively write
as much as possible. Ideally, the code would be shared between
the send routines and the progress engine. */
rc = MPIDU_Sock_writev(vcch->sock, iov, n_iov, &nb);
if (rc == MPI_SUCCESS)
{
int offset = 0;
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
while (offset < n_iov)
{
if ((int)iov[offset].MPID_IOV_LEN <= nb)
{
nb -= iov[offset].MPID_IOV_LEN;
offset++;
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"partial write, request enqueued at head");
update_request(sreq, iov, n_iov, offset, nb);
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(vcch->conn->sock,
sreq->dev.iov + offset,
sreq->dev.iov_count - offset, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
break;
}
}
if (offset == n_iov)
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"write complete, calling OnDataAvail fcn");
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
}
else {
int complete;
mpi_errno = reqFn( vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (!complete) {
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_writev(
vcch->conn->sock, sreq->dev.iov,
sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
}
/* --END ERROR HANDLING-- */
}
}
}
}
/* --BEGIN ERROR HANDLING-- */
else if (MPIR_ERR_GET_CLASS(rc) == MPIDU_SOCK_ERR_NOMEM)
{
MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,
"MPIDU_Sock_writev failed, out of memory");
sreq->status.MPI_ERROR = MPIR_ERR_MEMALLOCFAILED;
}
else
{
MPIU_DBG_MSG_D(CH3_CHANNEL,TYPICAL,
"MPIDU_Sock_writev failed, rc=%d", rc);
/* Connection just failed. Mark the request complete and
return an error. */
MPIU_DBG_VCCHSTATECHANGE(vc,VC_STATE_FAILED);
/* FIXME: Shouldn't the vc->state also change? */
vcch->state = MPIDI_CH3I_VC_STATE_FAILED;
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* MT - CH3U_Request_complete performs write barrier */
MPIDI_CH3U_Request_complete(sreq);
/* Return error to calling routine */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"send queue not empty, enqueuing");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING)
{
/* queuing the data so it can be sent later. */
MPIU_DBG_VCUSE(vc,"connecting. Enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED)
{
/* Form a new connection, queuing the data so it can be sent later. */
MPIU_DBG_VCUSE(vc,"unconnected. Enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
mpi_errno = MPIDI_CH3I_VC_post_connect(vc);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPIU_DBG_VCUSE(vc,"still connecting. enqueuing request");
update_request(sreq, iov, n_iov, 0, 0);
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
MPIU_DBG_VCUSE(vc,"connection failed");
/* Connection failed. Mark the request complete and return an error. */
/* TODO: Create an appropriate error message */
sreq->status.MPI_ERROR = MPI_ERR_INTERN;
/* MT - CH3U_Request_complete performs write barrier */
MPIDI_CH3U_Request_complete(sreq);
}
/* --END ERROR HANDLING-- */
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_ISENDV);
return mpi_errno;
}
