int MPID_nem_tcp_pkt_unpause_handler(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, intptr_t *buflen, MPIR_Request **rreqp) { int mpi_errno = MPI_SUCCESS; MPID_nem_tcp_vc_area *vc_tcp = VC_TCP(vc); MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_TCP_CKPT_UNPAUSE_HANDLER); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_TCP_CKPT_UNPAUSE_HANDLER); vc_tcp->send_paused = FALSE; /* There may be a unpause message in the send queue. If so, just enqueue everything on the send queue. */ if (MPIDI_CH3I_Sendq_empty(vc_tcp->send_queue)) mpi_errno = MPID_nem_tcp_send_queued(vc, &vc_tcp->paused_send_queue); /* if anything is left on the paused queue, put it on the send queue and wait for the reconnect */ if (!MPIDI_CH3I_Sendq_empty(vc_tcp->paused_send_queue)) { MPIDI_CH3I_Sendq_enqueue_multiple_no_refcount(&vc_tcp->send_queue, vc_tcp->paused_send_queue.head, vc_tcp->paused_send_queue.tail); vc_tcp->paused_send_queue.head = vc_tcp->paused_send_queue.tail = NULL; } fn_exit: *buflen = sizeof(MPIDI_CH3_Pkt_t); MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_TCP_CKPT_UNPAUSE_HANDLER); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_vc_terminate(MPIDI_VC_t * vc) { int mpi_errno = MPI_SUCCESS; int req_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_VC_TERMINATE); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_VC_TERMINATE); if (vc->state != MPIDI_VC_STATE_CLOSED) { /* VC is terminated as a result of a fault. Complete * outstanding sends with an error and terminate connection * immediately. */ MPIU_ERR_SET1(req_errno, MPI_ERR_OTHER, "**comm_fail", "**comm_fail %d", vc->pg_rank); mpi_errno = MPID_nem_scif_error_out_send_queue(vc, req_errno); if (mpi_errno) MPIU_ERR_POP(mpi_errno); mpi_errno = MPID_nem_scif_vc_terminated(vc); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } else { MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); vc_scif->terminate = 1; /* VC is terminated as a result of the close protocol. Wait * for sends to complete, then terminate. */ if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* The sendq is empty, so we can immediately terminate the * connection. */ mpi_errno = MPID_nem_scif_vc_terminated(vc); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } /* else: just return. We'll call vc_terminated() from the * commrdy_handler once the sendq is empty. */ } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_VC_TERMINATE); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_error_out_send_queue(struct MPIDI_VC *const vc, int req_errno) { int mpi_errno = MPI_SUCCESS; MPID_Request *req; MPID_nem_scif_vc_area *const vc_scif = VC_SCIF(vc); MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ERROR_OUT_SEND_QUEUE); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ERROR_OUT_SEND_QUEUE); /* we don't call onDataAvail or onFinal handlers because this is * an error condition and we just want to mark them as complete */ /* send queue */ while (!MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { MPIDI_CH3I_Sendq_dequeue(&vc_scif->send_queue, &req); req->status.MPI_ERROR = req_errno; MPIDI_CH3U_Request_complete(req); } MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ERROR_OUT_SEND_QUEUE); return mpi_errno; }
/* MPIDI_CH3I_SendNoncontig - Sends a message by packing directly into cells. The caller must initialize sreq->dev.segment as well as segment_first and segment_size. */ int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz ) { int mpi_errno = MPI_SUCCESS; int again = 0; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header); MPIU_THREAD_CS_ENTER(MPIDCOMM,); if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */ { /* send queue is not empty, enqueue the request then check to see if we can send any now */ MPIDI_DBG_PRINTF((55, FCNAME, "enqueuing")); sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.noncontig = TRUE; sreq->ch.header_sz = hdr_sz; sreq->ch.vc = vc; MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); mpi_errno = MPIDI_CH3I_Shm_send_progress(); if (mpi_errno) MPIU_ERR_POP(mpi_errno); goto fn_exit; } /* send as many cells of data as you can */ MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, header, hdr_sz, vc, &again); while(!again && sreq->dev.segment_first < sreq->dev.segment_size) MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again); if (again) { /* we didn't finish sending everything */ sreq->ch.noncontig = TRUE; sreq->ch.vc = vc; if (sreq->dev.segment_first == 0) /* nothing was sent, save header */ { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.header_sz = hdr_sz; } else { /* part of message was sent, make this req an active send */ MPIU_Assert(MPIDI_CH3I_shm_active_send == NULL); MPIDI_CH3I_shm_active_send = sreq; } MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); goto fn_exit; } /* finished sending all data, complete the request */ if (!sreq->dev.OnDataAvail) { MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } else { int complete = 0; mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_Assert(complete); /* all data has been sent, we should always complete */ MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } fn_exit: MPIU_THREAD_CS_EXIT(MPIDCOMM,); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; int iov_n; MPID_IOV iov[MPID_IOV_LIMIT]; MPID_IOV *iov_p; MPIDI_msg_sz_t offset = 0; int complete; uint64_t seqno = 0; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_SendNoncontig"); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); iov[0].MPID_IOV_BUF = header; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); iov_n = MPID_IOV_LIMIT - 1; mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &iov[1], &iov_n); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**loadsendiov"); iov_n += 1; offset = 0; if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) && MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) { offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend, iov, iov_n, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "scif_send noncontig " MPIDI_MSG_SZ_FMT, offset); } if (offset < iov[0].MPID_IOV_LEN) { /* header was not yet sent, save it in req */ sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) header; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & sreq->dev.pending_pkt; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); } /* check if whole iov was sent, and save any unsent portion of * iov */ sreq->dev.iov_count = 0; complete = 1; for (iov_p = &iov[0]; iov_p < &iov[iov_n]; ++iov_p) { if (offset < iov_p->MPID_IOV_LEN) { sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) iov_p->MPID_IOV_BUF + offset); sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_LEN = iov_p->MPID_IOV_LEN - offset; offset = 0; ++sreq->dev.iov_count; complete = 0; seqno = 0; } else offset -= iov_p->MPID_IOV_LEN; } if (seqno) complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno); if (complete) { /* sent whole iov */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } complete = 0; mpi_errno = reqFn(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (complete) { MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } seqno = 0; } /* enqueue request */ MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing"); MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0)); RQ_SCIF(sreq)->seqno = seqno; sreq->ch.vc = vc; sreq->dev.iov_offset = 0; if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* this will be the first send on the queue: queue it and set * the write flag on the pollfd */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); } else { /* there are other sends in the queue before this one: try to * send from the queue */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG); return mpi_errno; fn_fail: MPIU_Object_set_ref(sreq, 0); MPIDI_CH3_Request_destroy(sreq); goto fn_exit; }
int MPID_nem_scif_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t offset = 0; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); scifconn_t *sc = vc_scif->sc; uint64_t seqno = 0; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iSendContig"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) && MPID_nem_scif_poll_send(sc->fd, &sc->csend)) { MPID_IOV iov[2]; iov[0].MPID_IOV_BUF = hdr; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); iov[1].MPID_IOV_BUF = data; iov[1].MPID_IOV_LEN = data_sz; offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d", offset, sc->fd)); if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) { /* sent whole message */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); if (seqno) goto enqueue_request; reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } else { int complete = 0; mpi_errno = reqFn(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (complete) { MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } /* not completed: more to send */ goto enqueue_request; } } } /* save iov */ if (offset < sizeof(MPIDI_CH3_Pkt_t)) { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr; sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset; sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset; if (data_sz) { sreq->dev.iov[1].MPID_IOV_BUF = data; sreq->dev.iov[1].MPID_IOV_LEN = data_sz; sreq->dev.iov_count = 2; } else sreq->dev.iov_count = 1; seqno = 0; } else { sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov_count = 1; seqno = 0; } enqueue_request: /* enqueue request */ RQ_SCIF(sreq)->seqno = seqno; MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing"); MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0)); sreq->ch.vc = vc; sreq->dev.iov_offset = 0; if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* this will be the first send on the queue: queue it and set * the write flag on the pollfd */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); } else { /* there are other sends in the queue before this one: try to * send from the queue */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_send_queued(MPIDI_VC_t * vc, reqq_t * send_queue) { int mpi_errno = MPI_SUCCESS; MPID_Request *sreq; MPIDI_msg_sz_t offset; MPID_IOV *iov; int complete; uint64_t seqno; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED); MPIU_Assert(vc != NULL); if (MPIDI_CH3I_Sendq_empty(*send_queue)) { if (vc_scif->terminate) MPID_nem_scif_vc_terminated(vc); goto fn_exit; } while (!MPIDI_CH3I_Sendq_empty(*send_queue)) { sreq = MPIDI_CH3I_Sendq_head(*send_queue); seqno = RQ_SCIF(sreq)->seqno; complete = 0; if (seqno) { /* We were just waiting for DMA to finish */ complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno); } else if (MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) { MPIU_DBG_MSG_P(CH3_CHANNEL, VERBOSE, "Sending %p", sreq); iov = &sreq->dev.iov[sreq->dev.iov_offset]; offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend, iov, sreq->dev.iov_count, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "write " MPIDI_MSG_SZ_FMT, offset); RQ_SCIF(sreq)->seqno = seqno; complete = 1; for (iov = &sreq->dev.iov[sreq->dev.iov_offset]; iov < &sreq->dev.iov[sreq->dev.iov_offset + sreq->dev.iov_count]; ++iov) { if (offset < iov->MPID_IOV_LEN) { iov->MPID_IOV_BUF = (char *) iov->MPID_IOV_BUF + offset; iov->MPID_IOV_LEN -= offset; /* iov_count should be equal to the number of iov's remaining */ sreq->dev.iov_count -= ((iov - sreq->dev.iov) - sreq->dev.iov_offset); sreq->dev.iov_offset = iov - sreq->dev.iov; complete = 0; seqno = 0; break; } offset -= iov->MPID_IOV_LEN; } if (seqno) complete = 0; } if (!complete) { /* writev couldn't write the entire iov, give up for now */ break; } else { /* sent whole message */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); RQ_SCIF(sreq)->seqno = 0; reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq); continue; } complete = 0; mpi_errno = reqFn(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (complete) { MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); MPIDI_CH3I_Sendq_dequeue(send_queue, &sreq); continue; } sreq->dev.iov_offset = 0; } } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SEND_QUEUED); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr) { int mpi_errno = MPI_SUCCESS; MPID_Request *sreq = NULL; MPIDI_msg_sz_t offset = 0; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); scifconn_t *sc = vc_scif->sc; uint64_t seqno = 0; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iStartContigMsg"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) && MPID_nem_scif_poll_send(sc->fd, &sc->csend)) { MPID_IOV iov[2]; iov[0].MPID_IOV_BUF = hdr; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); iov[1].MPID_IOV_BUF = data; iov[1].MPID_IOV_LEN = data_sz; offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d", offset, sc->fd)); if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) { if (!seqno) { /* sent whole message */ *sreq_ptr = NULL; goto fn_exit; } } } /* create and enqueue request */ MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing"); /* create a request */ sreq = MPID_Request_create(); RQ_SCIF(sreq)->seqno = seqno; MPIU_Assert(sreq != NULL); MPIU_Object_set_ref(sreq, 2); sreq->kind = MPID_REQUEST_SEND; sreq->dev.OnDataAvail = 0; sreq->ch.vc = vc; sreq->dev.iov_offset = 0; if (!seqno) { if (offset < sizeof(MPIDI_CH3_Pkt_t)) { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr; sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset; sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset; if (data_sz) { sreq->dev.iov[1].MPID_IOV_BUF = data; sreq->dev.iov[1].MPID_IOV_LEN = data_sz; sreq->dev.iov_count = 2; } else sreq->dev.iov_count = 1; } else { sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov_count = 1; } MPIU_Assert(sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0); } if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* this will be the first send on the queue: queue it and set * the write flag on the pollfd */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); } else { /* there are other sends in the queue before this one: try to * send from the queue */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } *sreq_ptr = sreq; fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
/* MPIDI_CH3I_SendNoncontig - Sends a message by packing directly into cells. The caller must initialize sreq->dev.segment as well as segment_first and segment_size. */ int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPIR_Request *sreq, void *header, intptr_t hdr_sz, MPL_IOV *hdr_iov, int n_hdr_iov) { int mpi_errno = MPI_SUCCESS; int again = 0; intptr_t orig_segment_first = sreq->dev.segment_first; MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header); MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX); if (n_hdr_iov > 0) { /* translate segments to iovs and combine with the extended header iov. */ mpi_errno = MPIDI_CH3_SendNoncontig_iov(vc, sreq, header, hdr_sz, hdr_iov, n_hdr_iov); if (mpi_errno) MPIR_ERR_POP(mpi_errno); goto fn_exit; } if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */ { /* send queue is not empty, enqueue the request then check to see if we can send any now */ MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER, TERSE, "enqueuing"); sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.noncontig = TRUE; sreq->ch.header_sz = hdr_sz; sreq->ch.vc = vc; MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); mpi_errno = MPIDI_CH3I_Shm_send_progress(); if (mpi_errno) MPIR_ERR_POP(mpi_errno); goto fn_exit; } /* send as many cells of data as you can */ MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, header, hdr_sz, vc, &again); while(!again && sreq->dev.segment_first < sreq->dev.segment_size) MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again); if (again) { /* we didn't finish sending everything */ sreq->ch.noncontig = TRUE; sreq->ch.vc = vc; if (sreq->dev.segment_first == orig_segment_first) /* nothing was sent, save header */ { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.header_sz = hdr_sz; } else { /* part of message was sent, make this req an active send */ MPIR_Assert(MPIDI_CH3I_shm_active_send == NULL); MPIDI_CH3I_shm_active_send = sreq; } MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); goto fn_exit; } /* finished sending all data, complete the request */ if (!sreq->dev.OnDataAvail) { MPIR_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); mpi_errno = MPID_Request_complete(sreq); if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP(mpi_errno); } MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } else { int complete = 0; mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete); if (mpi_errno) MPIR_ERR_POP(mpi_errno); MPIR_Assert(complete); /* all data has been sent, we should always complete */ MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } fn_exit: MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX); MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); return mpi_errno; fn_fail: goto fn_exit; }