/* MPIDI_CH3I_SendNoncontig - Sends a message by packing directly into cells. The caller must initialize sreq->dev.segment as well as segment_first and segment_size. */ int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPID_Request *sreq, void *header, MPIDI_msg_sz_t hdr_sz ) { int mpi_errno = MPI_SUCCESS; int again = 0; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header); MPIU_THREAD_CS_ENTER(MPIDCOMM,); if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */ { /* send queue is not empty, enqueue the request then check to see if we can send any now */ MPIDI_DBG_PRINTF((55, FCNAME, "enqueuing")); sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.noncontig = TRUE; sreq->ch.header_sz = hdr_sz; sreq->ch.vc = vc; MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); mpi_errno = MPIDI_CH3I_Shm_send_progress(); if (mpi_errno) MPIU_ERR_POP(mpi_errno); goto fn_exit; } /* send as many cells of data as you can */ MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, header, hdr_sz, vc, &again); while(!again && sreq->dev.segment_first < sreq->dev.segment_size) MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again); if (again) { /* we didn't finish sending everything */ sreq->ch.noncontig = TRUE; sreq->ch.vc = vc; if (sreq->dev.segment_first == 0) /* nothing was sent, save header */ { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.header_sz = hdr_sz; } else { /* part of message was sent, make this req an active send */ MPIU_Assert(MPIDI_CH3I_shm_active_send == NULL); MPIDI_CH3I_shm_active_send = sreq; } MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); goto fn_exit; } /* finished sending all data, complete the request */ if (!sreq->dev.OnDataAvail) { MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } else { int complete = 0; mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); MPIU_Assert(complete); /* all data has been sent, we should always complete */ MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } fn_exit: MPIU_THREAD_CS_EXIT(MPIDCOMM,); MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_SendNoncontig(MPIDI_VC_t * vc, MPID_Request * sreq, void *header, MPIDI_msg_sz_t hdr_sz) { int mpi_errno = MPI_SUCCESS; int iov_n; MPID_IOV iov[MPID_IOV_LIMIT]; MPID_IOV *iov_p; MPIDI_msg_sz_t offset = 0; int complete; uint64_t seqno = 0; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_SendNoncontig"); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); iov[0].MPID_IOV_BUF = header; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); iov_n = MPID_IOV_LIMIT - 1; mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &iov[1], &iov_n); MPIU_ERR_CHKANDJUMP(mpi_errno, mpi_errno, MPI_ERR_OTHER, "**loadsendiov"); iov_n += 1; offset = 0; if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) && MPID_nem_scif_poll_send(vc_scif->sc->fd, &vc_scif->sc->csend)) { offset = MPID_nem_scif_writev(vc_scif->sc->fd, &vc_scif->sc->csend, iov, iov_n, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "scif_send noncontig " MPIDI_MSG_SZ_FMT, offset); } if (offset < iov[0].MPID_IOV_LEN) { /* header was not yet sent, save it in req */ sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) header; iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) & sreq->dev.pending_pkt; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); } /* check if whole iov was sent, and save any unsent portion of * iov */ sreq->dev.iov_count = 0; complete = 1; for (iov_p = &iov[0]; iov_p < &iov[iov_n]; ++iov_p) { if (offset < iov_p->MPID_IOV_LEN) { sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_BUF = (MPID_IOV_BUF_CAST) ((char *) iov_p->MPID_IOV_BUF + offset); sreq->dev.iov[sreq->dev.iov_count].MPID_IOV_LEN = iov_p->MPID_IOV_LEN - offset; offset = 0; ++sreq->dev.iov_count; complete = 0; seqno = 0; } else offset -= iov_p->MPID_IOV_LEN; } if (seqno) complete = MPID_nem_scif_chk_seqno(&vc_scif->sc->csend, seqno); if (complete) { /* sent whole iov */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } complete = 0; mpi_errno = reqFn(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (complete) { MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } seqno = 0; } /* enqueue request */ MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing"); MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0)); RQ_SCIF(sreq)->seqno = seqno; sreq->ch.vc = vc; sreq->dev.iov_offset = 0; if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* this will be the first send on the queue: queue it and set * the write flag on the pollfd */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); } else { /* there are other sends in the queue before this one: try to * send from the queue */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_SENDNONCONTIG); return mpi_errno; fn_fail: MPIU_Object_set_ref(sreq, 0); MPIDI_CH3_Request_destroy(sreq); goto fn_exit; }
int MPID_nem_scif_iStartContigMsg(MPIDI_VC_t * vc, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz, MPID_Request ** sreq_ptr) { int mpi_errno = MPI_SUCCESS; MPID_Request *sreq = NULL; MPIDI_msg_sz_t offset = 0; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); scifconn_t *sc = vc_scif->sc; uint64_t seqno = 0; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iStartContigMsg"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) && MPID_nem_scif_poll_send(sc->fd, &sc->csend)) { MPID_IOV iov[2]; iov[0].MPID_IOV_BUF = hdr; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); iov[1].MPID_IOV_BUF = data; iov[1].MPID_IOV_LEN = data_sz; offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d", offset, sc->fd)); if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) { if (!seqno) { /* sent whole message */ *sreq_ptr = NULL; goto fn_exit; } } } /* create and enqueue request */ MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing"); /* create a request */ sreq = MPID_Request_create(); RQ_SCIF(sreq)->seqno = seqno; MPIU_Assert(sreq != NULL); MPIU_Object_set_ref(sreq, 2); sreq->kind = MPID_REQUEST_SEND; sreq->dev.OnDataAvail = 0; sreq->ch.vc = vc; sreq->dev.iov_offset = 0; if (!seqno) { if (offset < sizeof(MPIDI_CH3_Pkt_t)) { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr; sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset; sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset; if (data_sz) { sreq->dev.iov[1].MPID_IOV_BUF = data; sreq->dev.iov[1].MPID_IOV_LEN = data_sz; sreq->dev.iov_count = 2; } else sreq->dev.iov_count = 1; } else { sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov_count = 1; } MPIU_Assert(sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0); } if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* this will be the first send on the queue: queue it and set * the write flag on the pollfd */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); } else { /* there are other sends in the queue before this one: try to * send from the queue */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } *sreq_ptr = sreq; fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISTARTCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
int MPID_nem_scif_iSendContig(MPIDI_VC_t * vc, MPID_Request * sreq, void *hdr, MPIDI_msg_sz_t hdr_sz, void *data, MPIDI_msg_sz_t data_sz) { int mpi_errno = MPI_SUCCESS; MPIDI_msg_sz_t offset = 0; MPID_nem_scif_vc_area *vc_scif = VC_SCIF(vc); scifconn_t *sc = vc_scif->sc; uint64_t seqno = 0; MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG); MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG); MPIU_Assert(hdr_sz <= sizeof(MPIDI_CH3_Pkt_t)); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "scif_iSendContig"); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *) hdr); if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue) && MPID_nem_scif_poll_send(sc->fd, &sc->csend)) { MPID_IOV iov[2]; iov[0].MPID_IOV_BUF = hdr; iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t); iov[1].MPID_IOV_BUF = data; iov[1].MPID_IOV_LEN = data_sz; offset = MPID_nem_scif_writev(sc->fd, &sc->csend, iov, 2, &seqno); MPIU_ERR_CHKANDJUMP1(offset <= 0, mpi_errno, MPI_ERR_OTHER, "**scif_writev", "**scif_writev %s", MPIU_Strerror(errno)); MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST, "scif_send " MPIDI_MSG_SZ_FMT " fd=%d", offset, sc->fd)); if (offset == sizeof(MPIDI_CH3_Pkt_t) + data_sz) { /* sent whole message */ int (*reqFn) (MPIDI_VC_t *, MPID_Request *, int *); if (seqno) goto enqueue_request; reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } else { int complete = 0; mpi_errno = reqFn(vc, sreq, &complete); if (mpi_errno) MPIU_ERR_POP(mpi_errno); if (complete) { MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete"); goto fn_exit; } /* not completed: more to send */ goto enqueue_request; } } } /* save iov */ if (offset < sizeof(MPIDI_CH3_Pkt_t)) { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *) hdr; sreq->dev.iov[0].MPID_IOV_BUF = (char *) &sreq->dev.pending_pkt + offset; sreq->dev.iov[0].MPID_IOV_LEN = sizeof(MPIDI_CH3_Pkt_t) - offset; if (data_sz) { sreq->dev.iov[1].MPID_IOV_BUF = data; sreq->dev.iov[1].MPID_IOV_LEN = data_sz; sreq->dev.iov_count = 2; } else sreq->dev.iov_count = 1; seqno = 0; } else { sreq->dev.iov[0].MPID_IOV_BUF = (char *) data + (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov[0].MPID_IOV_LEN = data_sz - (offset - sizeof(MPIDI_CH3_Pkt_t)); sreq->dev.iov_count = 1; seqno = 0; } enqueue_request: /* enqueue request */ RQ_SCIF(sreq)->seqno = seqno; MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "enqueuing"); MPIU_Assert(seqno || (sreq->dev.iov_count >= 1 && sreq->dev.iov[0].MPID_IOV_LEN > 0)); sreq->ch.vc = vc; sreq->dev.iov_offset = 0; if (MPIDI_CH3I_Sendq_empty(vc_scif->send_queue)) { /* this will be the first send on the queue: queue it and set * the write flag on the pollfd */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); } else { /* there are other sends in the queue before this one: try to * send from the queue */ MPIDI_CH3I_Sendq_enqueue(&vc_scif->send_queue, sreq); mpi_errno = MPID_nem_scif_send_queued(vc, &vc_scif->send_queue); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_ISENDCONTIGMSG); return mpi_errno; fn_fail: goto fn_exit; }
/* MPIDI_CH3I_SendNoncontig - Sends a message by packing directly into cells. The caller must initialize sreq->dev.segment as well as segment_first and segment_size. */ int MPIDI_CH3I_SendNoncontig( MPIDI_VC_t *vc, MPIR_Request *sreq, void *header, intptr_t hdr_sz, MPL_IOV *hdr_iov, int n_hdr_iov) { int mpi_errno = MPI_SUCCESS; int again = 0; intptr_t orig_segment_first = sreq->dev.segment_first; MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)header); MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX); if (n_hdr_iov > 0) { /* translate segments to iovs and combine with the extended header iov. */ mpi_errno = MPIDI_CH3_SendNoncontig_iov(vc, sreq, header, hdr_sz, hdr_iov, n_hdr_iov); if (mpi_errno) MPIR_ERR_POP(mpi_errno); goto fn_exit; } if (!MPIDI_CH3I_Sendq_empty(MPIDI_CH3I_shm_sendq)) /* MT */ { /* send queue is not empty, enqueue the request then check to see if we can send any now */ MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER, TERSE, "enqueuing"); sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.noncontig = TRUE; sreq->ch.header_sz = hdr_sz; sreq->ch.vc = vc; MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); mpi_errno = MPIDI_CH3I_Shm_send_progress(); if (mpi_errno) MPIR_ERR_POP(mpi_errno); goto fn_exit; } /* send as many cells of data as you can */ MPID_nem_mpich_send_seg_header(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, header, hdr_sz, vc, &again); while(!again && sreq->dev.segment_first < sreq->dev.segment_size) MPID_nem_mpich_send_seg(sreq->dev.segment_ptr, &sreq->dev.segment_first, sreq->dev.segment_size, vc, &again); if (again) { /* we didn't finish sending everything */ sreq->ch.noncontig = TRUE; sreq->ch.vc = vc; if (sreq->dev.segment_first == orig_segment_first) /* nothing was sent, save header */ { sreq->dev.pending_pkt = *(MPIDI_CH3_Pkt_t *)header; sreq->ch.header_sz = hdr_sz; } else { /* part of message was sent, make this req an active send */ MPIR_Assert(MPIDI_CH3I_shm_active_send == NULL); MPIDI_CH3I_shm_active_send = sreq; } MPIDI_CH3I_Sendq_enqueue(&MPIDI_CH3I_shm_sendq, sreq); goto fn_exit; } /* finished sending all data, complete the request */ if (!sreq->dev.OnDataAvail) { MPIR_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP); mpi_errno = MPID_Request_complete(sreq); if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP(mpi_errno); } MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } else { int complete = 0; mpi_errno = sreq->dev.OnDataAvail(vc, sreq, &complete); if (mpi_errno) MPIR_ERR_POP(mpi_errno); MPIR_Assert(complete); /* all data has been sent, we should always complete */ MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete %d bytes", (int) (sreq->dev.segment_size)); } fn_exit: MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX); MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_SENDNONCONTIG); return mpi_errno; fn_fail: goto fn_exit; }