int MPIC_Recv(void *buf, MPI_Aint count, MPI_Datatype datatype, int source, int tag,
MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPI_Status mystatus;
MPID_Request *request_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_RECV);
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", count);
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
if (status == MPI_STATUS_IGNORE)
status = &mystatus;
mpi_errno = MPID_Recv(buf, count, datatype, source, tag, comm_ptr,
context_id, status, &request_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr, errflag);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
*status = request_ptr->status;
mpi_errno = status->MPI_ERROR;
MPID_Request_release(request_ptr);
} else {
MPIR_Process_status(status, errflag);
MPIR_TAG_CLEAR_ERROR_BITS(status->MPI_TAG);
}
if (MPI_SUCCESS == MPIR_ERR_GET_CLASS(status->MPI_ERROR)) {
MPIU_Assert(status->MPI_TAG == tag);
}
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_RECV);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (request_ptr) MPID_Request_release(request_ptr);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIC_Ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest, int tag,
MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPID_Request *request_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SSEND);
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", count);
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
switch(*errflag) {
case MPIR_ERR_NONE:
break;
case MPIR_ERR_PROC_FAILED:
MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
default:
MPIR_TAG_SET_ERROR_BIT(tag);
}
mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
context_id, &request_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPID_Request_release(request_ptr);
}
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SSEND);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (request_ptr) MPID_Request_release(request_ptr);
if (mpi_errno && !*errflag) {
if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
*errflag = MPIR_ERR_PROC_FAILED;
} else {
*errflag = MPIR_ERR_OTHER;
}
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_Nem_nd_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_INIT);
/* Initialize Executive */
mpi_errno = MPIU_ExInitialize();
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
/* Initialize ND state machine */
mpi_errno = MPID_Nem_nd_sm_init();
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
/* Listen for conns & create the business card */
mpi_errno = MPID_Nem_nd_listen_for_conn(pg_rank, bc_val_p, val_max_sz_p);
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_INIT);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_Nem_nd_vc_init(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_VC_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_VC_INIT);
vc->sendNoncontig_fn = MPID_Nem_nd_send_noncontig;
MPID_NEM_ND_VCCH_NETMOD_STATE_SET(vc, MPID_NEM_ND_VC_STATE_DISCONNECTED);
vc_ch->iStartContigMsg = MPID_Nem_nd_istart_contig_msg;
vc_ch->iSendContig = MPID_Nem_nd_send_contig;
vc_ch->next = NULL;
vc_ch->prev = NULL;
MPID_NEM_ND_VCCH_NETMOD_CONN_HND_INIT(vc);
MPID_NEM_ND_VCCH_NETMOD_TMP_CONN_HND_INIT(vc);
MPID_NEM_ND_VCCH_NETMOD_POSTED_SENDQ_INIT(vc);
MPID_NEM_ND_VCCH_NETMOD_PENDING_SENDQ_INIT(vc);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_VC_INIT);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
/*@
MPIDI_CH3U_VC_WaitForClose - Wait for all virtual connections to close
@*/
int MPIDI_CH3U_VC_WaitForClose( void )
{
MPID_Progress_state progress_state;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_VC_WAITFORCLOSE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_VC_WAITFORCLOSE);
MPID_Progress_start(&progress_state);
while(MPIDI_Outstanding_close_ops > 0) {
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL,
"Waiting for %d close operations",
MPIDI_Outstanding_close_ops);
mpi_errno = MPID_Progress_wait(&progress_state);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SET(mpi_errno,MPI_ERR_OTHER,"**ch3|close_progress");
break;
}
/* --END ERROR HANDLING-- */
}
MPID_Progress_end(&progress_state);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_VC_WAITFORCLOSE);
return mpi_errno;
}
int MPID_Nem_nd_vc_terminate (MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_VC_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_VC_TERMINATE);
/* Poll till no more pending/posted sends */
while(!MPID_NEM_ND_VCCH_NETMOD_POSTED_SENDQ_EMPTY(vc)
|| !MPID_NEM_ND_VCCH_NETMOD_PENDING_SENDQ_EMPTY(vc)){
mpi_errno = MPID_Nem_nd_sm_poll(1);
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
vc_ch->next = NULL;
vc_ch->prev = NULL;
MPID_NEM_ND_VCCH_NETMOD_STATE_SET(vc, MPID_NEM_ND_VC_STATE_DISCONNECTED);
mpi_errno = MPID_Nem_nd_conn_disc(MPID_NEM_ND_VCCH_NETMOD_CONN_HND_GET(vc));
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPID_NEM_ND_VCCH_NETMOD_CONN_HND_SET(vc, MPID_NEM_ND_CONN_HND_INVALID);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_VC_TERMINATE);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
static int handler_recv_dequeue_complete(const ptl_event_t *e)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *const rreq = e->user_ptr;
int is_contig;
MPI_Aint last;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype *dt_ptr ATTRIBUTE((unused));
MPIDI_STATE_DECL(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIDI_FUNC_ENTER(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
MPIU_Assert(e->type == PTL_EVENT_PUT || e->type == PTL_EVENT_PUT_OVERFLOW);
MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, is_contig, data_sz, dt_ptr, dt_true_lb);
dequeue_req(e);
if (e->type == PTL_EVENT_PUT_OVERFLOW) {
/* unpack the data from unexpected buffer */
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "is_contig = %d", is_contig);
if (is_contig) {
MPIU_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, e->start, e->mlength);
} else {
last = e->mlength;
MPID_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, &last, e->start);
if (last != e->mlength)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
} else {
/* Data was placed directly into the user buffer, so datatype mismatch
is harder to detect. We use a simple check ensuring the received bytes
are a multiple of a single basic element. Currently, we do not detect
mismatches with datatypes constructed of more than one basic type */
MPI_Datatype dt_basic_type;
MPID_Datatype_get_basic_type(rreq->dev.datatype, dt_basic_type);
if (dt_basic_type != MPI_DATATYPE_NULL && (e->mlength % MPID_Datatype_get_basic_size(dt_basic_type)) != 0)
MPIR_ERR_SET(rreq->status.MPI_ERROR, MPI_ERR_TYPE, "**dtypemismatch");
}
mpi_errno = handler_recv_complete(e);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_HANDLER_RECV_DEQUEUE_COMPLETE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Nem_nd_vc_destroy(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_VC_DESTROY);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_VC_DESTROY);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_VC_DESTROY);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_Nem_nd_connect_to_root(const char *business_card, MPIDI_VC_t *new_vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_CONNECT_TO_ROOT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_CONNECT_TO_ROOT);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_CONNECT_TO_ROOT);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_Nem_nd_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_GET_BUSINESS_CARD);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_GET_BUSINESS_CARD);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_GET_BUSINESS_CARD);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPID_Nem_nd_ckpt_shutdown()
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_CKPT_SHUTDOWN);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_CKPT_SHUTDOWN);
mpi_errno = MPID_Nem_nd_finalize();
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_CKPT_SHUTDOWN);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
void MPIDI_CH3I_Comm_find(MPIR_Context_id_t context_id, MPID_Comm **comm)
{
MPIDI_STATE_DECL(MPIDI_STATE_MPIDI_CH3I_COMM_FIND);
MPIDI_FUNC_ENTER(MPIDI_STATE_MPIDI_CH3I_COMM_FIND);
COMM_FOREACH((*comm)) {
if ((*comm)->context_id == context_id || ((*comm)->context_id + MPID_CONTEXT_INTRA_COLL) == context_id ||
((*comm)->node_comm && ((*comm)->node_comm->context_id == context_id || ((*comm)->node_comm->context_id + MPID_CONTEXT_INTRA_COLL) == context_id)) ||
((*comm)->node_roots_comm && ((*comm)->node_roots_comm->context_id == context_id || ((*comm)->node_roots_comm->context_id + MPID_CONTEXT_INTRA_COLL) == context_id)) ) {
MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,"Found matching context id: %d", (*comm)->context_id);
break;
}
}
MPIDI_FUNC_EXIT(MPIDI_STATE_MPIDI_CH3I_COMM_FIND);
}
int MPIC_Sendrecv_replace(void *buf, int count, MPI_Datatype datatype,
int dest, int sendtag,
int source, int recvtag,
MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
MPI_Status mystatus;
MPIR_Context_id_t context_id_offset;
MPID_Request *sreq = NULL;
MPID_Request *rreq = NULL;
void *tmpbuf = NULL;
MPI_Aint tmpbuf_size = 0;
MPI_Aint tmpbuf_count = 0;
MPIU_CHKLMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV_REPLACE);
#ifdef MPID_LOG_ARROWS
/* The logging macros log sendcount and recvcount */
int sendcount = count, recvcount = count;
#endif
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV_REPLACE);
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", count);
if (status == MPI_STATUS_IGNORE) status = &mystatus;
switch(*errflag) {
case MPIR_ERR_NONE:
break;
case MPIR_ERR_PROC_FAILED:
MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
default:
MPIR_TAG_SET_ERROR_BIT(sendtag);
}
context_id_offset = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
if (count > 0 && dest != MPI_PROC_NULL) {
MPIR_Pack_size_impl(count, datatype, &tmpbuf_size);
MPIU_CHKLMEM_MALLOC(tmpbuf, void *, tmpbuf_size, mpi_errno, "temporary send buffer");
mpi_errno = MPIR_Pack_impl(buf, count, datatype, tmpbuf, tmpbuf_size, &tmpbuf_count);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
int MPID_Nem_nd_finalize()
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_FINALIZE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_FINALIZE);
/* Finalize the ND state machine */
MPID_Nem_nd_sm_finalize();
/* Finalize the executive */
MPIU_ExFinalize();
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_FINALIZE);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPIC_Wait(MPID_Request * request_ptr, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIC_WAIT);
MPIDI_PT2PT_FUNC_ENTER(MPID_STATE_MPIC_WAIT);
MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");
if (request_ptr->kind == MPID_REQUEST_SEND)
request_ptr->status.MPI_TAG = 0;
if (!MPID_Request_is_complete(request_ptr))
{
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (!MPID_Request_is_complete(request_ptr))
{
mpi_errno = MPID_Progress_wait(&progress_state);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
MPID_Progress_end(&progress_state);
}
if (request_ptr->kind == MPID_REQUEST_RECV)
MPIR_Process_status(&request_ptr->status, errflag);
MPIR_TAG_CLEAR_ERROR_BITS(request_ptr->status.MPI_TAG);
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_PT2PT_FUNC_EXIT(MPID_STATE_MPIC_WAIT);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_nem_scif_send_queued(MPIDI_VC_t * vc, reqq_t * send_queue)
{
int mpi_errno = MPI_SUCCESS;
MPID_Request *sreq;
MPIDI_msg_sz_t offset;
MPID_IOV *iov;
int complete;
uint64_t seqno;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
MPIU_Assert(vc != NULL);
if (MPIDI_CH3I_Sendq_empty(*send_queue)) {
if (vc_scif->terminate)
MPID_nem_scif_vc_terminated(vc);
goto fn_exit;
}
while (!MPIDI_CH3I_Sendq_empty(*send_queue)) {
sreq = MPIDI_CH3I_Sendq_head(*send_queue);
seqno = RQ_SCIF(sreq)->seqno;
complete = 0;
if (seqno) {
/* We were just waiting for DMA to finish */
complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);
}
else if (MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "Sending %p", sreq);
iov = &sreq->dev.iov[sreq->dev.iov_offset];
offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend,
iov, sreq->dev.iov_count, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write " MPIDI_MSG_SZ_FMT, offset);
RQ_SCIF(sreq)->seqno = seqno;
complete = 1;
for (iov = &sreq->dev.iov[sreq->dev.iov_offset];
iov < &sreq->dev.iov[sreq->dev.iov_offset + sreq->dev.iov_count]; ++iov) {
if (offset < iov->MPID_IOV_LEN) {
iov->MPID_IOV_BUF = (char *) iov->MPID_IOV_BUF + offset;
iov->MPID_IOV_LEN -= offset;
/* iov_count should be equal to the number of iov's remaining */
sreq->dev.iov_count -= ((iov - sreq->dev.iov) - sreq->dev.iov_offset);
sreq->dev.iov_offset = iov - sreq->dev.iov;
complete = 0;
seqno = 0;
break;
}
offset -= iov->MPID_IOV_LEN;
}
if (seqno)
complete = 0;
}
if (!complete) {
/* writev couldn't write the entire iov, give up for now */
break;
}
else {
/* sent whole message */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
RQ_SCIF(sreq)->seqno = 0;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
}
complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq);
continue;
}
sreq->dev.iov_offset = 0;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPIDI_CH3U_Handle_connection - handle connection event
Input Parameters:
+ vc - virtual connection
. event - connection event
NOTE:
This routine is used to transition the VC state.
The only events currently handled are TERMINATED events. This
routine should be called (with TERMINATED) whenever a connection is
terminated whether normally (in MPIDI_CH3_Connection_terminate() ),
or abnormally.
FIXME: Currently state transitions resulting from receiving CLOSE
packets are performed in MPIDI_CH3_PktHandler_Close(). Perhaps that
should move here.
@*/
int MPIDI_CH3U_Handle_connection(MPIDI_VC_t * vc, MPIDI_VC_Event_t event)
{
int inuse;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
switch (event)
{
case MPIDI_VC_EVENT_TERMINATED:
{
switch (vc->state)
{
case MPIDI_VC_STATE_CLOSED:
/* Normal termination. */
MPIDI_CHANGE_VC_STATE(vc, INACTIVE);
/* MT: this is not thread safe */
MPIDI_Outstanding_close_ops -= 1;
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL,
"outstanding close operations = %d", MPIDI_Outstanding_close_ops);
if (MPIDI_Outstanding_close_ops == 0)
{
MPIDI_CH3_Progress_signal_completion();
mpi_errno = MPIDI_CH3_Channel_close();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
break;
case MPIDI_VC_STATE_INACTIVE:
/* VC was terminated before it was activated.
This can happen if a failed process was
detected before the process used the VC. */
MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL, "VC terminated before it was activated. We probably got a failed"
" process notification.");
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
break;
case MPIDI_VC_STATE_ACTIVE:
case MPIDI_VC_STATE_REMOTE_CLOSE:
/* This is a premature termination. This process
has not started the close protocol. There may
be outstanding sends or receives on the local
side, remote side or both. */
MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL, "Connection closed prematurely.");
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDU_Ftb_publish_vc(MPIDU_FTB_EV_UNREACHABLE, vc);
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
break;
case MPIDI_VC_STATE_LOCAL_CLOSE:
/* This is a premature termination. This process
has started the close protocol, but hasn't
received a CLOSE packet from the remote side.
This process may not have been able to send the
CLOSE ack=F packet to the remote side. There
may be outstanding sends or receives on the
local or remote sides. */
case MPIDI_VC_STATE_CLOSE_ACKED:
/* This is a premature termination. Both sides
have started the close protocol. This process
has received CLOSE ack=F, but not CLOSE ack=t.
This process may not have been able to send
CLOSE ack=T. There should not be any
outstanding sends or receives on either
side. */
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL, "Connection closed prematurely during close protocol. "
"Outstanding close operations = %d", MPIDI_Outstanding_close_ops);
MPIDI_CH3U_Complete_posted_with_error(vc);
++MPIDI_Failed_vc_count;
MPIDU_Ftb_publish_vc(MPIDU_FTB_EV_UNREACHABLE, vc);
MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
/* MT: this is not thread safe */
MPIDI_Outstanding_close_ops -= 1;
if (MPIDI_Outstanding_close_ops == 0) {
MPIDI_CH3_Progress_signal_completion();
mpi_errno = MPIDI_CH3_Channel_close();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
break;
default:
{
MPIU_DBG_MSG_D(CH3_DISCONNECT,TYPICAL, "Unhandled connection state %d when closing connection",vc->state);
mpi_errno = MPIR_Err_create_code(
MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|unhandled_connection_state",
"**ch3|unhandled_connection_state %p %d", vc, vc->state);
goto fn_fail;
break;
}
}
/* FIXME: Decrement the reference count? Who increments? */
/* FIXME: The reference count is often already 0. But
not always */
/* MPIU_Object_set_ref(vc, 0); ??? */
/*
* FIXME: The VC used in connect accept has a NULL
* process group
*/
/* XXX DJG FIXME-MT should we be checking this ref_count? */
if (vc->pg != NULL && (MPIU_Object_get_ref(vc) == 0))
{
/* FIXME: Who increments the reference count that
this is decrementing? */
/* When the reference count for a vc becomes zero,
decrement the reference count
of the associated process group. */
/* FIXME: This should be done when the reference
count of the vc is first decremented */
MPIDI_PG_release_ref(vc->pg, &inuse);
if (inuse == 0) {
MPIDI_PG_Destroy(vc->pg);
}
}
break;
}
default:
{
break;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_HANDLE_CONNECTION);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIC_Sendrecv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype,
int dest, int sendtag, void *recvbuf, MPI_Aint recvcount,
MPI_Datatype recvtype, int source, int recvtag,
MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPI_Status mystatus;
MPID_Request *recv_req_ptr = NULL, *send_req_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV);
MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");
MPIU_ERR_CHKANDJUMP1((sendcount < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", sendcount);
MPIU_ERR_CHKANDJUMP1((recvcount < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", recvcount);
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
if (status == MPI_STATUS_IGNORE) status = &mystatus;
switch(*errflag) {
case MPIR_ERR_NONE:
break;
case MPIR_ERR_PROC_FAILED:
MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
default:
MPIR_TAG_SET_ERROR_BIT(sendtag);
}
mpi_errno = MPID_Irecv(recvbuf, recvcount, recvtype, source, recvtag,
comm_ptr, context_id, &recv_req_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_Isend(sendbuf, sendcount, sendtype, dest, sendtag,
comm_ptr, context_id, &send_req_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIC_Wait(send_req_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIC_Wait(recv_req_ptr, errflag);
if (mpi_errno) MPIU_ERR_POPFATAL(mpi_errno);
*status = recv_req_ptr->status;
if (mpi_errno == MPI_SUCCESS) {
mpi_errno = recv_req_ptr->status.MPI_ERROR;
if (mpi_errno == MPI_SUCCESS) {
mpi_errno = send_req_ptr->status.MPI_ERROR;
}
}
MPID_Request_release(send_req_ptr);
MPID_Request_release(recv_req_ptr);
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SENDRECV);
return mpi_errno;
fn_fail:
if (send_req_ptr)
MPID_Request_release(send_req_ptr);
if (recv_req_ptr)
MPID_Request_release(recv_req_ptr);
goto fn_exit;
}
PMPI_LOCAL int MPIR_Comm_create_calculate_mapping(MPID_Group *group_ptr,
MPID_Comm *comm_ptr,
MPID_VCR **mapping_vcr_out,
int **mapping_out)
{
int mpi_errno = MPI_SUCCESS;
int subsetOfWorld = 0;
int i, j;
int n;
int *mapping=0;
int vcr_size;
MPID_VCR *vcr;
MPIU_CHKPMEM_DECL(1);
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_CREATE_CALCULATE_MAPPING);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_CREATE_CALCULATE_MAPPING);
*mapping_out = NULL;
*mapping_vcr_out = NULL;
/* N.B. For intracomms only the comm_ptr->vcr is valid and populated,
* however local_size and remote_size are always set to the same value for
* intracomms. For intercomms both are valid and populated, with the
* local_vcr holding VCs corresponding to the local_group, local_comm, and
* local_size.
*
* For this mapping calculation we always want the logically local vcr,
* regardless of whether it is stored in the "plain" vcr or local_vcr. */
if (comm_ptr->comm_kind == MPID_INTERCOMM) {
vcr = comm_ptr->local_vcr;
vcr_size = comm_ptr->local_size;
}
else {
vcr = comm_ptr->vcr;
vcr_size = comm_ptr->remote_size;
}
n = group_ptr->size;
MPIU_CHKPMEM_MALLOC(mapping,int*,n*sizeof(int),mpi_errno,"mapping");
/* Make sure that the processes for this group are contained within
the input communicator. Also identify the mapping from the ranks of
the old communicator to the new communicator.
We do this by matching the lpids of the members of the group
with the lpids of the members of the input communicator.
It is an error if the group contains a reference to an lpid that
does not exist in the communicator.
An important special case is groups (and communicators) that
are subsets of MPI_COMM_WORLD. In this case, the lpids are
exactly the same as the ranks in comm world.
*/
/* we examine the group's lpids in both the intracomm and non-comm_world cases */
MPIR_Group_setup_lpid_list( group_ptr );
/* Optimize for groups contained within MPI_COMM_WORLD. */
if (comm_ptr->comm_kind == MPID_INTRACOMM) {
int wsize;
subsetOfWorld = 1;
wsize = MPIR_Process.comm_world->local_size;
for (i=0; i<n; i++) {
int g_lpid = group_ptr->lrank_to_lpid[i].lpid;
/* This mapping is relative to comm world */
MPIU_DBG_MSG_FMT(COMM,VERBOSE,
(MPIU_DBG_FDEST,
"comm-create - mapping into world[%d] = %d\n",
i, g_lpid ));
if (g_lpid < wsize) {
mapping[i] = g_lpid;
}
else {
subsetOfWorld = 0;
break;
}
}
}
MPIU_DBG_MSG_D(COMM,VERBOSE, "subsetOfWorld=%d", subsetOfWorld );
if (subsetOfWorld) {
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
int idx;
mpi_errno = MPIR_GroupCheckVCRSubset( group_ptr, vcr_size, vcr, &idx );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Override the vcr to be used with the mapping array. */
vcr = MPIR_Process.comm_world->vcr;
vcr_size = MPIR_Process.comm_world->local_size;
}
else {
for (i=0; i<n; i++) {
/* mapping[i] is the rank in the communicator of the process
that is the ith element of the group */
/* FIXME : BUBBLE SORT */
mapping[i] = -1;
for (j=0; j<vcr_size; j++) {
int comm_lpid;
MPID_VCR_Get_lpid( vcr[j], &comm_lpid );
if (comm_lpid == group_ptr->lrank_to_lpid[i].lpid) {
mapping[i] = j;
break;
}
}
MPIU_ERR_CHKANDJUMP1(mapping[i] == -1,mpi_errno,MPI_ERR_GROUP,
"**groupnotincomm", "**groupnotincomm %d", i );
}
}
MPIU_Assert(vcr != NULL);
MPIU_Assert(mapping != NULL);
*mapping_vcr_out = vcr;
*mapping_out = mapping;
MPL_VG_CHECK_MEM_IS_DEFINED(*mapping_vcr_out, vcr_size * sizeof(**mapping_vcr_out));
MPL_VG_CHECK_MEM_IS_DEFINED(*mapping_out, n * sizeof(**mapping_out));
MPIU_CHKPMEM_COMMIT();
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_COMM_CREATE_CALCULATE_MAPPING);
return mpi_errno;
fn_fail:
MPIU_CHKPMEM_REAP();
goto fn_exit;
}
void MPIR_Free_contextid(MPIU_Context_id_t context_id)
{
int idx, bitpos, raw_prefix;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_FREE_CONTEXTID);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_FREE_CONTEXTID);
/* Convert the context id to the bit position */
raw_prefix = MPID_CONTEXT_READ_FIELD(PREFIX, context_id);
idx = raw_prefix / MPIR_CONTEXT_INT_BITS;
bitpos = raw_prefix % MPIR_CONTEXT_INT_BITS;
/* --BEGIN ERROR HANDLING-- */
if (idx < 0 || idx >= MPIR_MAX_CONTEXT_MASK) {
MPID_Abort(0, MPI_ERR_INTERN, 1, "In MPIR_Free_contextid, idx is out of range");
}
/* --END ERROR HANDLING-- */
/* The low order bits for dynamic context IDs don't have meaning the
* same way that low bits of non-dynamic ctx IDs do. So we have to
* check the dynamic case first. */
if (MPID_CONTEXT_READ_FIELD(DYNAMIC_PROC, context_id)) {
MPIU_DBG_MSG_D(COMM, VERBOSE, "skipping dynamic process ctx id, context_id=%d", context_id);
goto fn_exit;
}
else { /* non-dynamic context ID */
/* In terms of the context ID bit vector, intercomms and their constituent
* localcomms have the same value. To avoid a double-free situation we just
* don't free the context ID for localcomms and assume it will be cleaned up
* when the parent intercomm is itself completely freed. */
if (MPID_CONTEXT_READ_FIELD(IS_LOCALCOMM, context_id)) {
#ifdef USE_DBG_LOGGING
char dump_str[1024];
dump_context_id(context_id, dump_str, sizeof(dump_str));
MPIU_DBG_MSG_S(COMM, VERBOSE, "skipping localcomm id: %s", dump_str);
#endif
goto fn_exit;
}
else if (MPID_CONTEXT_READ_FIELD(SUBCOMM, context_id)) {
MPIU_DBG_MSG_D(COMM, VERBOSE, "skipping non-parent communicator ctx id, context_id=%d",
context_id);
goto fn_exit;
}
}
/* --BEGIN ERROR HANDLING-- */
/* Check that this context id has been allocated */
if ((context_mask[idx] & (0x1 << bitpos)) != 0) {
#ifdef USE_DBG_LOGGING
char dump_str[1024];
dump_context_id(context_id, dump_str, sizeof(dump_str));
MPIU_DBG_MSG_S(COMM, VERBOSE, "context dump: %s", dump_str);
MPIU_DBG_MSG_S(COMM, VERBOSE, "context mask = %s", context_mask_to_str());
#endif
MPID_Abort(0, MPI_ERR_INTERN, 1, "In MPIR_Free_contextid, the context id is not in use");
}
/* --END ERROR HANDLING-- */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
/* MT: Note that this update must be done atomically in the multithreaedd
* case. In the "one, single lock" implementation, that lock is indeed
* held when this operation is called. */
context_mask[idx] |= (0x1 << bitpos);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
MPIU_DBG_MSG_FMT(COMM, VERBOSE,
(MPIU_DBG_FDEST,
"Freed context %d, mask[%d] bit %d (prefix=%#x)",
context_id, idx, bitpos, raw_prefix));
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_FREE_CONTEXTID);
}
int MPIR_Get_contextid_sparse_group(MPID_Comm * comm_ptr, MPID_Group * group_ptr, int tag,
MPIU_Context_id_t * context_id, int ignore_id)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
struct gcn_state st;
struct gcn_state *tmp;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_GET_CONTEXTID);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_GET_CONTEXTID);
st.first_iter = 1;
st.comm_ptr = comm_ptr;
st.tag = tag;
st.own_mask = 0;
st.own_eager_mask = 0;
/* Group-collective and ignore_id should never be combined */
MPIU_Assert(!(group_ptr != NULL && ignore_id));
*context_id = 0;
MPIU_DBG_MSG_FMT(COMM, VERBOSE, (MPIU_DBG_FDEST,
"Entering; shared state is %d:%d, my ctx id is %d, tag=%d",
mask_in_use, eager_in_use, comm_ptr->context_id, tag));
while (*context_id == 0) {
/* We lock only around access to the mask (except in the global locking
* case). If another thread is using the mask, we take a mask of zero. */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
if (initialize_context_mask) {
context_id_init();
}
if (eager_nelem < 0) {
/* Ensure that at least one word of deadlock-free context IDs is
* always set aside for the base protocol */
MPIU_Assert(MPIR_CVAR_CTXID_EAGER_SIZE >= 0 &&
MPIR_CVAR_CTXID_EAGER_SIZE < MPIR_MAX_CONTEXT_MASK - 1);
eager_nelem = MPIR_CVAR_CTXID_EAGER_SIZE;
}
if (ignore_id) {
/* We are not participating in the resulting communicator, so our
* context ID space doesn't matter. Set the mask to "all available". */
memset(st.local_mask, 0xff, MPIR_MAX_CONTEXT_MASK * sizeof(int));
st.own_mask = 0;
/* don't need to touch mask_in_use/lowest_context_id b/c our thread
* doesn't ever need to "win" the mask */
}
/* Deadlock avoidance: Only participate in context id loop when all
* processes have called this routine. On the first iteration, use the
* "eager" allocation protocol.
*/
else if (st.first_iter) {
memset(st.local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int));
st.own_eager_mask = 0;
/* Attempt to reserve the eager mask segment */
if (!eager_in_use && eager_nelem > 0) {
int i;
for (i = 0; i < eager_nelem; i++)
st.local_mask[i] = context_mask[i];
eager_in_use = 1;
st.own_eager_mask = 1;
}
}
else {
MPIU_Assert(next_gcn != NULL);
/*If we are here, at least one element must be in the list, at least myself */
/* only the first element in the list can own the mask. However, maybe the mask is used
* by another thread, which added another allcoation to the list bevore. So we have to check,
* if the mask is used and mark, if we own it */
if (mask_in_use || &st != next_gcn) {
memset(st.local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int));
st.own_mask = 0;
MPIU_DBG_MSG_FMT(COMM, VERBOSE, (MPIU_DBG_FDEST,
"Mask is in use, my context_id is %d, owner context id is %d",
st.comm_ptr->context_id,
next_gcn->comm_ptr->context_id));
}
else {
int i;
/* Copy safe mask segment to local_mask */
for (i = 0; i < eager_nelem; i++)
st.local_mask[i] = 0;
for (i = eager_nelem; i < MPIR_MAX_CONTEXT_MASK; i++)
st.local_mask[i] = context_mask[i];
mask_in_use = 1;
st.own_mask = 1;
MPIU_DBG_MSG(COMM, VERBOSE, "Copied local_mask");
}
}
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
/* Note: MPIR_MAX_CONTEXT_MASK elements of local_mask are used by the
* context ID allocation algorithm. The additional element is ignored
* by the context ID mask access routines and is used as a flag for
* detecting context ID exhaustion (explained below). */
if (st.own_mask || ignore_id)
st.local_mask[ALL_OWN_MASK_FLAG] = 1;
else
st.local_mask[ALL_OWN_MASK_FLAG] = 0;
/* Now, try to get a context id */
MPIU_Assert(comm_ptr->comm_kind == MPID_INTRACOMM);
/* In the global and brief-global cases, note that this routine will
* release that global lock when it needs to wait. That will allow
* other processes to enter the global or brief global critical section.
*/
if (group_ptr != NULL) {
int coll_tag = tag | MPIR_Process.tagged_coll_mask; /* Shift tag into the tagged coll space */
mpi_errno = MPIR_Allreduce_group(MPI_IN_PLACE, st.local_mask, MPIR_MAX_CONTEXT_MASK + 1,
MPI_INT, MPI_BAND, comm_ptr, group_ptr, coll_tag,
&errflag);
}
else {
mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, st.local_mask, MPIR_MAX_CONTEXT_MASK + 1,
MPI_INT, MPI_BAND, comm_ptr, &errflag);
}
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
/* MT FIXME 2/3 cases don't seem to need the CONTEXTID CS, check and
* narrow this region */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
if (ignore_id) {
/* we don't care what the value was, but make sure that everyone
* who did care agreed on a value */
*context_id = locate_context_bit(st.local_mask);
/* used later in out-of-context ids check and outer while loop condition */
}
else if (st.own_eager_mask) {
/* There is a chance that we've found a context id */
/* Find_and_allocate_context_id updates the context_mask if it finds a match */
*context_id = find_and_allocate_context_id(st.local_mask);
MPIU_DBG_MSG_D(COMM, VERBOSE, "Context id is now %hd", *context_id);
st.own_eager_mask = 0;
eager_in_use = 0;
if (*context_id <= 0) {
/* else we did not find a context id. Give up the mask in case
* there is another thread (with a lower input context id)
* waiting for it. We need to ensure that any other threads
* have the opportunity to run, hence yielding */
/* FIXME: Do we need to do an GLOBAL yield here?
* When we do a collective operation, we anyway yield
* for other others */
MPID_THREAD_CS_YIELD(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX); /* FG: TODO Double-check */
MPID_THREAD_CS_YIELD(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
}
else if (st.own_mask) {
/* There is a chance that we've found a context id */
/* Find_and_allocate_context_id updates the context_mask if it finds a match */
*context_id = find_and_allocate_context_id(st.local_mask);
MPIU_DBG_MSG_D(COMM, VERBOSE, "Context id is now %hd", *context_id);
mask_in_use = 0;
if (*context_id > 0) {
/* If we found a new context id, we have to remove the element from the list, so the
* next allocation can own the mask */
if (next_gcn == &st) {
next_gcn = st.next;
}
else {
for (tmp = next_gcn; tmp->next != &st; tmp = tmp->next); /* avoid compiler warnings */
tmp->next = st.next;
}
}
else {
/* else we did not find a context id. Give up the mask in case
* there is another thread in the gcn_next_list
* waiting for it. We need to ensure that any other threads
* have the opportunity to run, hence yielding */
/* FIXME: Do we need to do an GLOBAL yield here?
* When we do a collective operation, we anyway yield
* for other others */
MPID_THREAD_CS_YIELD(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPID_THREAD_CS_YIELD(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
}
else {
/* As above, force this thread to yield */
/* FIXME: Do we need to do an GLOBAL yield here? When we
* do a collective operation, we anyway yield for other
* others */
MPID_THREAD_CS_YIELD(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPID_THREAD_CS_YIELD(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
/* Test for context ID exhaustion: All threads that will participate in
* the new communicator owned the mask and could not allocate a context
* ID. This indicates that either some process has no context IDs
* available, or that some are available, but the allocation cannot
* succeed because there is no common context ID. */
if (*context_id == 0 && st.local_mask[ALL_OWN_MASK_FLAG] == 1) {
/* --BEGIN ERROR HANDLING-- */
int nfree = 0;
int ntotal = 0;
int minfree;
if (st.own_mask) {
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
mask_in_use = 0;
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
context_mask_stats(&nfree, &ntotal);
if (ignore_id)
minfree = INT_MAX;
else
minfree = nfree;
if (group_ptr != NULL) {
int coll_tag = tag | MPIR_Process.tagged_coll_mask; /* Shift tag into the tagged coll space */
mpi_errno = MPIR_Allreduce_group(MPI_IN_PLACE, &minfree, 1, MPI_INT, MPI_MIN,
comm_ptr, group_ptr, coll_tag, &errflag);
}
else {
mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, &minfree, 1, MPI_INT,
MPI_MIN, comm_ptr, &errflag);
}
if (minfree > 0) {
MPIR_ERR_SETANDJUMP3(mpi_errno, MPI_ERR_OTHER,
"**toomanycommfrag", "**toomanycommfrag %d %d %d",
nfree, ntotal, ignore_id);
}
else {
MPIR_ERR_SETANDJUMP3(mpi_errno, MPI_ERR_OTHER,
"**toomanycomm", "**toomanycomm %d %d %d",
nfree, ntotal, ignore_id);
}
/* --END ERROR HANDLING-- */
}
if (st.first_iter == 1) {
st.first_iter = 0;
/* to avoid deadlocks, the element is not added to the list bevore the first iteration */
if (!ignore_id && *context_id == 0)
add_gcn_to_list(&st);
}
}
fn_exit:
if (ignore_id)
*context_id = MPIU_INVALID_CONTEXT_ID;
MPIU_DBG_MSG_S(COMM, VERBOSE, "Context mask = %s", context_mask_to_str());
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_GET_CONTEXTID);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
/* Release the masks */
if (st.own_mask) {
/* is it safe to access this without holding the CS? */
mask_in_use = 0;
}
/*If in list, remove it */
if (!st.first_iter && !ignore_id) {
if (next_gcn == &st) {
next_gcn = st.next;
}
else {
for (tmp = next_gcn; tmp->next != &st; tmp = tmp->next);
tmp->next = st.next;
}
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
static int send_msg(ptl_hdr_data_t ssend_flag, struct MPIDI_VC *vc, const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest,
int tag, MPID_Comm *comm, int context_offset, struct MPID_Request **request)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
int ret;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
MPID_Request *sreq = NULL;
ptl_me_t me;
int initial_iov_count, remaining_iov_count;
ptl_md_t md;
MPI_Aint last;
MPIU_CHKPMEM_DECL(2);
MPIDI_STATE_DECL(MPID_STATE_SEND_MSG);
MPIDI_FUNC_ENTER(MPID_STATE_SEND_MSG);
MPID_nem_ptl_request_create_sreq(sreq, mpi_errno, comm);
sreq->dev.match.parts.rank = dest;
sreq->dev.match.parts.tag = tag;
sreq->dev.match.parts.context_id = comm->context_id + context_offset;
sreq->ch.vc = vc;
if (!vc_ptl->id_initialized) {
mpi_errno = MPID_nem_ptl_init_id(vc);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "count="MPI_AINT_FMT_DEC_SPEC" datatype=%#x contig=%d data_sz=%lu", count, datatype, dt_contig, data_sz));
if (data_sz <= PTL_LARGE_THRESHOLD) {
/* Small message. Send all data eagerly */
if (dt_contig) {
void *start = (char *)buf + dt_true_lb;
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Small contig message");
REQ_PTL(sreq)->event_handler = handler_send;
MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "&REQ_PTL(sreq)->event_handler = %p", &(REQ_PTL(sreq)->event_handler));
if (start == NULL)
ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)&dummy, data_sz, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, data_sz));
else
ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)start, data_sz, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, data_sz));
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", data_sz, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag, data_sz));
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "id.nid = %#x", vc_ptl->id.phys.nid);
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "id.pid = %#x", vc_ptl->id.phys.pid);
MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "sreq = %p", sreq);
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "vc_ptl->pt = %d", vc_ptl->pt);
MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "REQ_PTL(sreq)->event_handler = %p", REQ_PTL(sreq)->event_handler);
goto fn_exit;
}
/* noncontig data */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Small noncontig message");
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIR_ERR_CHKANDJUMP1(sreq->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
last = sreq->dev.segment_size;
sreq->dev.iov_count = MPL_IOV_LIMIT;
MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.iov, &sreq->dev.iov_count);
if (last == sreq->dev.segment_size) {
/* IOV is able to describe entire message */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, " entire message fits in IOV");
md.start = sreq->dev.iov;
md.length = sreq->dev.iov_count;
md.options = PTL_IOVEC;
md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(sreq)->md);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
REQ_PTL(sreq)->event_handler = handler_send;
ret = MPID_nem_ptl_rptl_put(REQ_PTL(sreq)->md, 0, data_sz, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, data_sz));
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("sreq", data_sz, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag, data_sz));
goto fn_exit;
}
/* IOV is not long enough to describe entire message */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, " IOV too long: using bounce buffer");
MPIU_CHKPMEM_MALLOC(REQ_PTL(sreq)->chunk_buffer[0], void *, data_sz, mpi_errno, "chunk_buffer");
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
last = data_sz;
MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, REQ_PTL(sreq)->chunk_buffer[0]);
MPIU_Assert(last == sreq->dev.segment_size);
REQ_PTL(sreq)->event_handler = handler_send;
ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)REQ_PTL(sreq)->chunk_buffer[0], data_sz, PTL_NO_ACK_REQ,
vc_ptl->id, vc_ptl->pt, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag, data_sz));
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", data_sz, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag, data_sz));
goto fn_exit;
}
/* Large message. Send first chunk of data and let receiver get the rest */
if (dt_contig) {
/* create ME for buffer so receiver can issue a GET for the data */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Large contig message");
big_meappend((char *)buf + dt_true_lb + PTL_LARGE_THRESHOLD, data_sz - PTL_LARGE_THRESHOLD, vc,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), sreq);
REQ_PTL(sreq)->event_handler = handler_send;
ret = MPID_nem_ptl_rptl_put(MPIDI_nem_ptl_global_md, (ptl_size_t)((char *)buf + dt_true_lb), PTL_LARGE_THRESHOLD, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz));
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("global", PTL_LARGE_THRESHOLD, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz));
goto fn_exit;
}
/* Large noncontig data */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "Large noncontig message");
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIR_ERR_CHKANDJUMP1(sreq->dev.segment_ptr == NULL, mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(buf, count, datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
last = PTL_LARGE_THRESHOLD;
sreq->dev.iov_count = MPL_IOV_LIMIT;
MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, &last, sreq->dev.iov, &sreq->dev.iov_count);
initial_iov_count = sreq->dev.iov_count;
sreq->dev.segment_first = last;
if (last == PTL_LARGE_THRESHOLD) {
/* first chunk of message fits into IOV */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, " first chunk fits in IOV");
if (initial_iov_count < MPL_IOV_LIMIT) {
/* There may be space for the rest of the message in this IOV */
sreq->dev.iov_count = MPL_IOV_LIMIT - sreq->dev.iov_count;
last = sreq->dev.segment_size;
MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, &last,
&sreq->dev.iov[initial_iov_count], &sreq->dev.iov_count);
remaining_iov_count = sreq->dev.iov_count;
if (last == sreq->dev.segment_size && last <= MPIDI_nem_ptl_ni_limits.max_msg_size + PTL_LARGE_THRESHOLD) {
/* Entire message fit in one IOV */
int was_incomplete;
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, " rest of message fits in one IOV");
/* Create ME for remaining data */
me.start = &sreq->dev.iov[initial_iov_count];
me.length = remaining_iov_count;
me.ct_handle = PTL_CT_NONE;
me.uid = PTL_UID_ANY;
me.options = ( PTL_ME_OP_PUT | PTL_ME_OP_GET | PTL_ME_USE_ONCE | PTL_ME_IS_ACCESSIBLE | PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE | PTL_IOVEC );
me.match_id = vc_ptl->id;
me.match_bits = NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank);
me.ignore_bits = 0;
me.min_free = 0;
MPIU_CHKPMEM_MALLOC(REQ_PTL(sreq)->get_me_p, ptl_handle_me_t *, sizeof(ptl_handle_me_t), mpi_errno, "get_me_p");
ret = PtlMEAppend(MPIDI_nem_ptl_ni, MPIDI_nem_ptl_get_pt, &me, PTL_PRIORITY_LIST, sreq,
&REQ_PTL(sreq)->get_me_p[0]);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmeappend", "**ptlmeappend %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_MEAPPEND("CTL", vc->pg_rank, me, sreq);
/* increment the cc for the get operation */
MPIDI_CH3U_Request_increment_cc(sreq, &was_incomplete);
MPIU_Assert(was_incomplete);
/* Create MD for first chunk */
md.start = sreq->dev.iov;
md.length = initial_iov_count;
md.options = PTL_IOVEC;
md.eq_handle = MPIDI_nem_ptl_origin_eq;
md.ct_handle = PTL_CT_NONE;
ret = PtlMDBind(MPIDI_nem_ptl_ni, &md, &REQ_PTL(sreq)->md);
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlmdbind", "**ptlmdbind %s", MPID_nem_ptl_strerror(ret));
REQ_PTL(sreq)->event_handler = handler_send;
ret = MPID_nem_ptl_rptl_put(REQ_PTL(sreq)->md, 0, PTL_LARGE_THRESHOLD, PTL_NO_ACK_REQ, vc_ptl->id, vc_ptl->pt,
NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), 0, sreq,
NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz));
MPIR_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**ptlput", "**ptlput %s", MPID_nem_ptl_strerror(ret));
DBG_MSG_PUT("req", PTL_LARGE_THRESHOLD, vc->pg_rank, NPTL_MATCH(tag, comm->context_id + context_offset, comm->rank), NPTL_HEADER(ssend_flag | NPTL_LARGE, data_sz));
goto fn_exit;
}
/* MPIDI_CH3_RndvSend - Send a request to perform a rendezvous send */
int MPIDI_CH3_RndvSend( MPID_Request **sreq_p, const void * buf, int count,
MPI_Datatype datatype, int dt_contig, MPIDI_msg_sz_t data_sz,
MPI_Aint dt_true_lb,
int rank,
int tag, MPID_Comm * comm, int context_offset )
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_rndv_req_to_send_t * const rts_pkt = &upkt.rndv_req_to_send;
MPIDI_VC_t * vc;
MPID_Request * rts_sreq;
MPID_Request *sreq =*sreq_p;
int mpi_errno = MPI_SUCCESS;
MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,
"sending rndv RTS, data_sz=" MPIDI_MSG_SZ_FMT, data_sz);
sreq->dev.OnDataAvail = 0;
sreq->partner_request = NULL;
MPIDI_Pkt_init(rts_pkt, MPIDI_CH3_PKT_RNDV_REQ_TO_SEND);
rts_pkt->match.parts.rank = comm->rank;
rts_pkt->match.parts.tag = tag;
rts_pkt->match.parts.context_id = comm->context_id + context_offset;
rts_pkt->sender_req_id = sreq->handle;
rts_pkt->data_sz = data_sz;
MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(rts_pkt, seqnum);
MPIDI_Request_set_seqnum(sreq, seqnum);
MPIU_DBG_MSGPKT(vc,tag,rts_pkt->match.parts.context_id,rank,data_sz,"Rndv");
MPIU_THREAD_CS_ENTER(CH3COMM,vc);
mpi_errno = MPIU_CALL(MPIDI_CH3,iStartMsg(vc, rts_pkt, sizeof(*rts_pkt),
&rts_sreq));
MPIU_THREAD_CS_EXIT(CH3COMM,vc);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rtspkt");
}
/* --END ERROR HANDLING-- */
if (rts_sreq != NULL)
{
if (rts_sreq->status.MPI_ERROR != MPI_SUCCESS)
{
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
*sreq_p = NULL;
mpi_errno = rts_sreq->status.MPI_ERROR;
MPID_Request_release(rts_sreq);
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ch3|rtspkt");
}
MPID_Request_release(rts_sreq);
}
/* FIXME: fill temporary IOV or pack temporary buffer after send to hide
some latency. This requires synchronization
because the CTS packet could arrive and be processed before the above
iStartmsg completes (depending on the progress
engine, threads, etc.). */
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static void MPIR_Bsend_free_segment( MPIR_Bsend_data_t *p )
{
MPIR_Bsend_data_t *prev = p->prev, *avail = BsendBuffer.avail, *avail_prev;
MPIU_DBG_MSG_FMT(BSEND,TYPICAL,(MPIU_DBG_FDEST,
"Freeing bsend segment at %p of size %llu, next at %p",
p, (unsigned long long) p->size, ((char *)p)+p->total_size));
MPIU_DBG_MSG_D(BSEND,TYPICAL,
"At the begining of free_segment with size %llu:",
(unsigned long long) p->total_size );
MPIU_DBG_STMT(BSEND,TYPICAL,MPIR_Bsend_dump());
/* Remove the segment from the active list */
if (prev) {
MPIU_DBG_MSG(BSEND,TYPICAL,"free segment is within active list");
prev->next = p->next;
}
else {
/* p was at the head of the active list */
MPIU_DBG_MSG(BSEND,TYPICAL,"free segment is head of active list");
BsendBuffer.active = p->next;
/* The next test sets the prev pointer to null */
}
if (p->next) {
p->next->prev = prev;
}
MPIU_DBG_STMT(BSEND,VERBOSE,MPIR_Bsend_dump());
/* Merge into the avail list */
/* Find avail_prev, avail, such that p is between them.
either may be null if p is at either end of the list */
avail_prev = 0;
while (avail) {
if (avail > p) {
break;
}
avail_prev = avail;
avail = avail->next;
}
/* Try to merge p with the next block */
if (avail) {
if ((char *)p + p->total_size == (char *)avail) {
p->total_size += avail->total_size;
p->size = p->total_size - BSENDDATA_HEADER_TRUE_SIZE;
p->next = avail->next;
if (avail->next) avail->next->prev = p;
avail = 0;
}
else {
p->next = avail;
avail->prev = p;
}
}
else {
p->next = 0;
}
/* Try to merge p with the previous block */
if (avail_prev) {
if ((char *)avail_prev + avail_prev->total_size == (char *)p) {
avail_prev->total_size += p->total_size;
avail_prev->size = avail_prev->total_size - BSENDDATA_HEADER_TRUE_SIZE;
avail_prev->next = p->next;
if (p->next) p->next->prev = avail_prev;
}
else {
avail_prev->next = p;
p->prev = avail_prev;
}
}
else {
/* p is the new head of the list */
BsendBuffer.avail = p;
p->prev = 0;
}
MPIU_DBG_MSG(BSEND,TYPICAL,"At the end of free_segment:" );
MPIU_DBG_STMT(BSEND,TYPICAL,MPIR_Bsend_dump());
}
static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,"Socket event of type %d", event->op_type );
switch (event->op_type)
{
case MPIDU_SOCK_OP_READ:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
MPID_Request * rreq = conn->recv_active;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS)
{
/* FIXME: the following should be handled by the close
protocol */
if (MPIR_ERR_GET_CLASS(event->error) != MPIDU_SOCK_ERR_CONN_CLOSED) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
break;
}
/* --END ERROR HANDLING-- */
if (conn->state == CONN_STATE_CONNECTED)
{
if (conn->recv_active == NULL)
{
MPIDI_msg_sz_t buflen = sizeof (MPIDI_CH3_Pkt_t);
MPIU_Assert(conn->pkt.type < MPIDI_CH3_PKT_END_CH3);
mpi_errno = pktArray[conn->pkt.type]( conn->vc, &conn->pkt,
&buflen, &rreq );
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
MPIU_Assert(buflen == sizeof (MPIDI_CH3_Pkt_t));
if (rreq == NULL)
{
if (conn->state != CONN_STATE_CLOSING)
{
/* conn->recv_active = NULL; --
already set to NULL */
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
}
else
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
else /* incoming data */
{
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = rreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(rreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, rreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
conn->recv_active = NULL;
mpi_errno = connection_post_recv_pkt(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to be read */
{
mpi_errno = ReadMoreData( conn, rreq );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
}
else if (conn->state == CONN_STATE_OPEN_LRECV_DATA)
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connopen_event( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
else /* Handling some internal connection establishment or
tear down packet */
{
mpi_errno = MPIDI_CH3_Sockconn_handle_conn_event( conn );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
break;
}
/* END OF SOCK_OP_READ */
case MPIDU_SOCK_OP_WRITE:
{
MPIDI_CH3I_Connection_t * conn =
(MPIDI_CH3I_Connection_t *) event->user_ptr;
/* --BEGIN ERROR HANDLING-- */
if (event->error != MPI_SUCCESS) {
mpi_errno = event->error;
MPIU_ERR_POP(mpi_errno);
}
/* --END ERROR HANDLING-- */
if (conn->send_active)
{
MPID_Request * sreq = conn->send_active;
int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *);
int complete;
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
else /* more data to send */
{
for(;;)
{
MPID_IOV * iovp;
MPIU_Size_t nb;
iovp = sreq->dev.iov;
mpi_errno = MPIDU_Sock_writev(conn->sock, iovp, sreq->dev.iov_count, &nb);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|immedwrite", "ch3|sock|immedwrite %p %p %p",
sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"immediate writev, vc=%p, sreq=0x%08x, nb=" MPIDI_MSG_SZ_FMT,
conn->vc, sreq->handle, nb));
if (nb > 0 && adjust_iov(&iovp, &sreq->dev.iov_count, nb))
{
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
complete = TRUE;
}
else {
mpi_errno = reqFn( conn->vc, sreq, &complete );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
if (complete)
{
mpi_errno = connection_pop_sendq_req(conn);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
break;
}
}
else
{
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting writev, vc=%p, conn=%p, sreq=0x%08x",
conn->vc, conn, sreq->handle));
mpi_errno = MPIDU_Sock_post_writev(conn->sock, iovp, sreq->dev.iov_count, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postwrite",
"ch3|sock|postwrite %p %p %p", sreq, conn, conn->vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
break;
}
}
}
}
else /* finished writing internal packet header */
{
/* the connection is not active yet */
mpi_errno = MPIDI_CH3_Sockconn_handle_connwrite( conn );
if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); }
}
break;
}
/* END OF SOCK_OP_WRITE */
case MPIDU_SOCK_OP_ACCEPT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_accept_event();
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CONNECT:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_connect_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr,
event->error );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_CLOSE:
{
mpi_errno = MPIDI_CH3_Sockconn_handle_close_event(
(MPIDI_CH3I_Connection_t *) event->user_ptr );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
break;
}
case MPIDU_SOCK_OP_WAKEUP:
{
MPIDI_CH3_Progress_signal_completion();
/* MPIDI_CH3I_progress_completion_count++; */
break;
}
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* MPIDI_CH3I_SendNoncontig - Sends a message by packing
directly into cells. The caller must initialize sreq->dev.segment
as well as segment_first and segment_size. */
int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz )
{
int mpi_errno = MPI_SUCCESS;
int again = 0;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header);
MPIU_THREAD_CS_ENTER(MPIDCOMM,);
if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */
{
/* send queue is not empty, enqueue the request then check to
see if we can send any now */
MPIDI_DBG_PRINTF((55, FCNAME, "enqueuing"));
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
sreq->ch.noncontig = TRUE;
sreq->ch.header_sz = hdr_sz;
sreq->ch.vc = vc;
MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq);
mpi_errno = MPIDI_CH3I_Shm_send_progress();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
goto fn_exit;
}
/* send as many cells of data as you can */
MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, header, hdr_sz, vc, &again);
while(!again && sreq->dev.segment_first < sreq->dev.segment_size)
MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again);
if (again)
{
/* we didn't finish sending everything */
sreq->ch.noncontig = TRUE;
sreq->ch.vc = vc;
if (sreq->dev.segment_first == 0) /* nothing was sent, save header */
{
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header;
sreq->ch.header_sz = hdr_sz;
}
else
{
/* part of message was sent, make this req an active send */
MPIU_Assert(MPIDI_CH3I_shm_active_send == NULL);
MPIDI_CH3I_shm_active_send = sreq;
}
MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq);
goto fn_exit;
}
/* finished sending all data, complete the request */
if (!sreq->dev.OnDataAvail)
{
MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
else
{
int complete = 0;
mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_Assert(complete); /* all data has been sent, we should always complete */
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size));
}
fn_exit:
MPIU_THREAD_CS_EXIT(MPIDCOMM,);
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Comm_commit(MPID_Comm * comm)
{
int mpi_errno = MPI_SUCCESS;
int num_local = -1, num_external = -1;
int local_rank = -1, external_rank = -1;
int *local_procs = NULL, *external_procs = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_COMMIT);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_COMMIT);
/* It's OK to relax these assertions, but we should do so very
* intentionally. For now this function is the only place that we create
* our hierarchy of communicators */
MPIU_Assert(comm->node_comm == NULL);
MPIU_Assert(comm->node_roots_comm == NULL);
mpi_errno = set_collops(comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Notify device of communicator creation */
mpi_errno = MPID_Dev_comm_create_hook(comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_Comm_map_free(comm);
if (comm->comm_kind == MPID_INTRACOMM) {
mpi_errno = MPIU_Find_local_and_external(comm,
&num_local, &local_rank, &local_procs,
&num_external, &external_rank, &external_procs,
&comm->intranode_table, &comm->internode_table);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno) {
if (MPIR_Err_is_fatal(mpi_errno))
MPIR_ERR_POP(mpi_errno);
/* Non-fatal errors simply mean that this communicator will not have
* any node awareness. Node-aware collectives are an optimization. */
MPIU_DBG_MSG_P(COMM, VERBOSE, "MPIU_Find_local_and_external failed for comm_ptr=%p",
comm);
if (comm->intranode_table)
MPIU_Free(comm->intranode_table);
if (comm->internode_table)
MPIU_Free(comm->internode_table);
mpi_errno = MPI_SUCCESS;
goto fn_exit;
}
/* --END ERROR HANDLING-- */
/* defensive checks */
MPIU_Assert(num_local > 0);
MPIU_Assert(num_local > 1 || external_rank >= 0);
MPIU_Assert(external_rank < 0 || external_procs != NULL);
/* if the node_roots_comm and comm would be the same size, then creating
* the second communicator is useless and wasteful. */
if (num_external == comm->remote_size) {
MPIU_Assert(num_local == 1);
goto fn_exit;
}
/* we don't need a local comm if this process is the only one on this node */
if (num_local > 1) {
mpi_errno = MPIR_Comm_create(&comm->node_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
comm->node_comm->context_id = comm->context_id + MPID_CONTEXT_INTRANODE_OFFSET;
comm->node_comm->recvcontext_id = comm->node_comm->context_id;
comm->node_comm->rank = local_rank;
comm->node_comm->comm_kind = MPID_INTRACOMM;
comm->node_comm->hierarchy_kind = MPID_HIERARCHY_NODE;
comm->node_comm->local_comm = NULL;
MPIU_DBG_MSG_D(CH3_OTHER, VERBOSE, "Create node_comm=%p\n", comm->node_comm);
comm->node_comm->local_size = num_local;
comm->node_comm->remote_size = num_local;
MPIR_Comm_map_irregular(comm->node_comm, comm, local_procs,
num_local, MPIR_COMM_MAP_DIR_L2L, NULL);
mpi_errno = set_collops(comm->node_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Notify device of communicator creation */
mpi_errno = MPID_Dev_comm_create_hook(comm->node_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* don't call MPIR_Comm_commit here */
MPIR_Comm_map_free(comm->node_comm);
}
/* this process may not be a member of the node_roots_comm */
if (local_rank == 0) {
mpi_errno = MPIR_Comm_create(&comm->node_roots_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
comm->node_roots_comm->context_id = comm->context_id + MPID_CONTEXT_INTERNODE_OFFSET;
comm->node_roots_comm->recvcontext_id = comm->node_roots_comm->context_id;
comm->node_roots_comm->rank = external_rank;
comm->node_roots_comm->comm_kind = MPID_INTRACOMM;
comm->node_roots_comm->hierarchy_kind = MPID_HIERARCHY_NODE_ROOTS;
comm->node_roots_comm->local_comm = NULL;
comm->node_roots_comm->local_size = num_external;
comm->node_roots_comm->remote_size = num_external;
MPIR_Comm_map_irregular(comm->node_roots_comm, comm,
external_procs, num_external, MPIR_COMM_MAP_DIR_L2L, NULL);
mpi_errno = set_collops(comm->node_roots_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* Notify device of communicator creation */
mpi_errno = MPID_Dev_comm_create_hook(comm->node_roots_comm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
/* don't call MPIR_Comm_commit here */
MPIR_Comm_map_free(comm->node_roots_comm);
}
comm->hierarchy_kind = MPID_HIERARCHY_PARENT;
}
fn_exit:
if (external_procs != NULL)
MPIU_Free(external_procs);
if (local_procs != NULL)
MPIU_Free(local_procs);
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_COMM_COMMIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_scif_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header,
MPIDI_msg_sz_t hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
int iov_n;
MPID_IOV iov[MPID_IOV_LIMIT];
MPID_IOV *iov_p;
MPIDI_msg_sz_t offset = 0;
int complete;
uint64_t seqno = 0;
MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_SendNoncontig");
MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
iov[0].MPID_IOV_BUF = header;
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
iov_n = MPID_IOV_LIMIT - 1;
mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &iov[1], &iov_n);
MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**loadsendiov");
iov_n += 1;
offset = 0;
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) &&
MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) {
offset =
MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend, iov, iov_n, &seqno);
MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER,
"**scif_writev", "**scif_writev %s", MPIU_Strerror(errno));
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "scif_send noncontig " MPIDI_MSG_SZ_FMT, offset);
}
if (offset < iov[0].MPID_IOV_LEN) {
/* header was not yet sent, save it in req */
sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) header;
iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & sreq->dev.pending_pkt;
iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t);
}
/* check if whole iov was sent, and save any unsent portion of
* iov */
sreq->dev.iov_count = 0;
complete = 1;
for (iov_p = &iov[0]; iov_p < &iov[iov_n]; ++iov_p) {
if (offset < iov_p->MPID_IOV_LEN) {
sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_BUF =
(MPID_IOV_BUF_CAST) ((char *) iov_p->MPID_IOV_BUF + offset);
sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_LEN = iov_p->MPID_IOV_LEN - offset;
offset = 0;
++sreq->dev.iov_count;
complete = 0;
seqno = 0;
}
else
offset -= iov_p->MPID_IOV_LEN;
}
if (seqno)
complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno);
if (complete) {
/* sent whole iov */
int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *);
reqFn = sreq->dev.OnDataAvail;
if (!reqFn) {
MPIDI_CH3U_Request_complete(sreq);
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
complete = 0;
mpi_errno = reqFn(vc, sreq, &complete);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (complete) {
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
goto fn_exit;
}
seqno = 0;
}
/* enqueue request */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing");
MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0));
RQ_SCIF(sreq)->seqno = seqno;
sreq->ch.vc = vc;
sreq->dev.iov_offset = 0;
if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) {
/* this will be the first send on the queue: queue it and set
* the write flag on the pollfd */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
}
else {
/* there are other sends in the queue before this one: try to
* send from the queue */
MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq);
mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG);
return mpi_errno;
fn_fail:
MPIU_Object_set_ref(sreq, 0);
MPIDI_CH3_Request_destroy(sreq);
goto fn_exit;
}
int MPIDI_CH3_iStartMsg(MPIDI_VC_t * vc, void * hdr, MPIDI_msg_sz_t hdr_sz,
MPID_Request ** sreq_ptr)
{
MPID_Request * sreq = NULL;
MPIDI_CH3I_VC *vcch = &vc->ch;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_ISTARTMSG);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_ISTARTMSG);
MPIU_Assert( hdr_sz <= sizeof(MPIDI_CH3_Pkt_t));
/* The SOCK channel uses a fixed length header, the size of which is the
maximum of all possible packet headers */
hdr_sz = sizeof(MPIDI_CH3_Pkt_t);
MPIU_DBG_STMT(CH3_CHANNEL,VERBOSE,
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t*)hdr));
if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTED) /* MT */
{
/* Connection already formed. If send queue is empty attempt to send
data, queuing any unsent data. */
if (MPIDI_CH3I_SendQ_empty(vcch)) /* MT */
{
MPIU_Size_t nb;
int rc;
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send queue empty, attempting to write");
MPIU_DBG_PKT(vcch->conn,hdr,"istartmsg");
/* MT: need some signalling to lock down our right to use the
channel, thus insuring that the progress engine does
not also try to write */
rc = MPIDU_Sock_write(vcch->sock, hdr, hdr_sz, &nb);
if (rc == MPI_SUCCESS)
{
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"wrote %ld bytes", (unsigned long) nb);
if (nb == hdr_sz)
{
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"entire write complete, " MPIDI_MSG_SZ_FMT " bytes", nb);
/* done. get us out of here as quickly as possible. */
}
else
{
MPIU_DBG_MSG_D(CH3_CHANNEL,VERBOSE,
"partial write of " MPIDI_MSG_SZ_FMT " bytes, request enqueued at head", nb);
sreq = create_request(hdr, hdr_sz, nb);
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue_head(vcch, sreq);
MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,
(MPIU_DBG_FDEST,"posting write, vc=0x%p, sreq=0x%08x", vc, sreq->handle));
vcch->conn->send_active = sreq;
mpi_errno = MPIDU_Sock_post_write(vcch->conn->sock, sreq->dev.iov[0].MPID_IOV_BUF,
sreq->dev.iov[0].MPID_IOV_LEN, sreq->dev.iov[0].MPID_IOV_LEN, NULL);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER,
"**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p",
sreq, vcch->conn, vc);
goto fn_fail;
}
/* --END ERROR HANDLING-- */
}
}
/* --BEGIN ERROR HANDLING-- */
else
{
MPIU_DBG_MSG_D(CH3_CHANNEL,TYPICAL,
"ERROR - MPIDU_Sock_write failed, rc=%d", rc);
sreq = MPID_Request_create();
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
sreq->kind = MPID_REQUEST_SEND;
MPID_cc_set(&(sreq->cc), 0);
sreq->status.MPI_ERROR = MPIR_Err_create_code( rc,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|writefailed",
"**ch3|sock|writefailed %d", rc );
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
}
else
{
MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
"send in progress, request enqueued");
sreq = create_request(hdr, hdr_sz, 0);
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_CONNECTING) /* MT */
{
MPIU_DBG_VCUSE(vc,
"connecteding. enqueuing request");
/* queue the data so it can be sent after the connection is formed */
sreq = create_request(hdr, hdr_sz, 0);
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
else if (vcch->state == MPIDI_CH3I_VC_STATE_UNCONNECTED) /* MT */
{
MPIU_DBG_VCUSE(vc,
"unconnected. posting connect and enqueuing request");
/* queue the data so it can be sent after the connection is formed */
sreq = create_request(hdr, hdr_sz, 0);
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
/* Form a new connection */
MPIDI_CH3I_VC_post_connect(vc);
}
else if (vcch->state != MPIDI_CH3I_VC_STATE_FAILED)
{
/* Unable to send data at the moment, so queue it for later */
MPIU_DBG_VCUSE(vc,"forming connection, request enqueued");
sreq = create_request(hdr, hdr_sz, 0);
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
MPIDI_CH3I_SendQ_enqueue(vcch, sreq);
}
/* --BEGIN ERROR HANDLING-- */
else
{
/* Connection failed, so allocate a request and return an error. */
MPIU_DBG_VCUSE(vc,"ERROR - connection failed");
sreq = MPID_Request_create();
if (!sreq) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
sreq->kind = MPID_REQUEST_SEND;
MPID_cc_set(&sreq->cc, 0);
sreq->status.MPI_ERROR = MPIR_Err_create_code( MPI_SUCCESS,
MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|sock|connectionfailed",0 );
/* Make sure that the caller sees this error */
mpi_errno = sreq->status.MPI_ERROR;
}
/* --END ERROR HANDLING-- */
fn_fail:
*sreq_ptr = sreq;
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_ISTARTMSG);
return mpi_errno;
}
int MPIR_Bsend_isend(const void *buf, int count, MPI_Datatype dtype,
int dest, int tag, MPID_Comm *comm_ptr,
MPIR_Bsend_kind_t kind, MPID_Request **request )
{
int mpi_errno = MPI_SUCCESS;
MPIR_Bsend_data_t *p;
MPIR_Bsend_msg_t *msg;
int packsize, pass;
/* Find a free segment and copy the data into it. If we could
have, we would already have used tBsend to send the message with
no copying.
We may want to decide here whether we need to pack at all
or if we can just use (a MPIU_Memcpy) of the buffer.
*/
/* We check the active buffer first. This helps avoid storage
fragmentation */
mpi_errno = MPIR_Bsend_check_active();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (dtype != MPI_PACKED)
MPIR_Pack_size_impl( count, dtype, &packsize );
else
packsize = count;
MPIU_DBG_MSG_D(BSEND,TYPICAL,"looking for buffer of size %d", packsize);
/*
* Use two passes. Each pass is the same; between the two passes,
* attempt to complete any active requests, and start any pending
* ones. If the message can be initiated in the first pass,
* do not perform the second pass.
*/
for (pass = 0; pass < 2; pass++) {
p = MPIR_Bsend_find_buffer( packsize );
if (p) {
MPIU_DBG_MSG_FMT(BSEND,TYPICAL,(MPIU_DBG_FDEST,
"found buffer of size %d with address %p",packsize,p));
/* Found a segment */
msg = &p->msg;
/* Pack the data into the buffer */
/* We may want to optimize for the special case of
either primative or contiguous types, and just
use MPIU_Memcpy and the provided datatype */
msg->count = 0;
if (dtype != MPI_PACKED)
{
mpi_errno = MPIR_Pack_impl( buf, count, dtype, p->msg.msgbuf, packsize, &p->msg.count);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else
{
MPIU_Memcpy(p->msg.msgbuf, buf, count);
p->msg.count = count;
}
/* Try to send the message. We must use MPID_Isend
because this call must not block */
mpi_errno = MPID_Isend(msg->msgbuf, msg->count, MPI_PACKED,
dest, tag, comm_ptr,
MPID_CONTEXT_INTRA_PT2PT, &p->request );
MPIU_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Bsend internal error: isend returned err");
/* If the error is "request not available", we should
put this on the pending list. This will depend on
how we signal failure to send. */
if (p->request) {
MPIU_DBG_MSG_FMT(BSEND,TYPICAL,
(MPIU_DBG_FDEST,"saving request %p in %p",p->request,p));
/* An optimization is to check to see if the
data has already been sent. The original code
to do this was commented out and probably did not match
the current request internals */
MPIR_Bsend_take_buffer( p, p->msg.count );
p->kind = kind;
*request = p->request;
}
break;
}
/* If we found a buffer or we're in the seccond pass, then break.
Note that the test on phere is redundant, as the code breaks
out of the loop in the test above if a block p is found. */
if (p || pass == 1) break;
MPIU_DBG_MSG(BSEND,TYPICAL,"Could not find storage, checking active");
/* Try to complete some pending bsends */
MPIR_Bsend_check_active( );
/* Give priority to any pending operations */
MPIR_Bsend_retry_pending( );
}
if (!p) {
/* Return error for no buffer space found */
/* Generate a traceback of the allocated space, explaining why
packsize could not be found */
MPIU_DBG_MSG(BSEND,TYPICAL,"Could not find space; dumping arena" );
MPIU_DBG_STMT(BSEND,TYPICAL,MPIR_Bsend_dump());
MPIU_ERR_SETANDJUMP2(mpi_errno, MPI_ERR_BUFFER, "**bufbsend", "**bufbsend %d %d", packsize, BsendBuffer.buffer_size);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
