int mca_btl_vader_put (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_base_segment_t *src = des->des_local; mca_btl_base_segment_t *dst = des->des_remote; const size_t size = min(dst->seg_len, src->seg_len); mca_mpool_base_registration_t *reg; void *rem_ptr; reg = vader_get_registation (endpoint, dst->seg_addr.pval, dst->seg_len, 0, &rem_ptr); if (OPAL_UNLIKELY(NULL == reg)) { return OPAL_ERROR; } vader_memmove (rem_ptr, src->seg_addr.pval, size); vader_return_registration (reg, endpoint); /* always call the callback function */ frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; mca_btl_vader_frag_complete (frag); return OPAL_SUCCESS; }
int mca_btl_vader_get (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_base_segment_t *src = des->des_remote; mca_btl_base_segment_t *dst = des->des_local; const size_t size = min(dst->seg_len, src->seg_len); struct iovec src_iov = {.iov_base = src->seg_addr.pval, .iov_len = size}; struct iovec dst_iov = {.iov_base = dst->seg_addr.pval, .iov_len = size}; ssize_t ret; ret = process_vm_readv (endpoint->seg_ds.seg_cpid, &dst_iov, 1, &src_iov, 1, 0); if (ret != (ssize_t)size) { opal_output(0, "Read %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno); return OPAL_ERROR; } /* always call the callback function */ frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; frag->endpoint = endpoint; mca_btl_vader_frag_complete (frag); return OPAL_SUCCESS; }
/** * Initiate a send to the peer. * * @param btl (IN) BTL module * @param peer (IN) BTL peer addressing */ int mca_btl_vader_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) descriptor; if (OPAL_LIKELY(frag->fbox)) { mca_btl_vader_fbox_send (frag->fbox, tag, frag->segments[0].seg_len); mca_btl_vader_frag_complete (frag); return 1; } /* header (+ optional inline data) */ frag->hdr->len = frag->segments[0].seg_len; /* type of message, pt-2-pt, one-sided, etc */ frag->hdr->tag = tag; /* post the relative address of the descriptor into the peer's fifo */ vader_fifo_write_ep (frag->hdr, endpoint); if ((frag->hdr->flags & MCA_BTL_VADER_FLAG_SINGLE_COPY) || !(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) { frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; return 0; } /* data is gone (from the pml's perspective). frag callback/release will happen later */ return 1; }
/** * Initiate a send to the peer. * * @param btl (IN) BTL module * @param peer (IN) BTL peer addressing */ int mca_btl_vader_send (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *descriptor, mca_btl_base_tag_t tag) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) descriptor; const size_t total_size = frag->segments[0].seg_len; if (OPAL_LIKELY(frag->fbox)) { mca_btl_vader_fbox_send (frag->fbox, tag); mca_btl_vader_frag_complete (frag); return 1; } /* header (+ optional inline data) */ frag->hdr->len = total_size; /* type of message, pt-2-pt, one-sided, etc */ frag->hdr->tag = tag; /* post the relative address of the descriptor into the peer's fifo */ if (opal_list_get_size (&endpoint->pending_frags) || !vader_fifo_write_ep (frag->hdr, endpoint)) { frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; OPAL_THREAD_LOCK(&endpoint->pending_frags_lock); opal_list_append (&endpoint->pending_frags, (opal_list_item_t *) frag); if (!endpoint->waiting) { OPAL_THREAD_LOCK(&mca_btl_vader_component.lock); opal_list_append (&mca_btl_vader_component.pending_endpoints, &endpoint->super); OPAL_THREAD_UNLOCK(&mca_btl_vader_component.lock); endpoint->waiting = true; } OPAL_THREAD_UNLOCK(&endpoint->pending_frags_lock); return OPAL_SUCCESS; } if ((frag->hdr->flags & MCA_BTL_VADER_FLAG_SINGLE_COPY) || !(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) { frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; return OPAL_SUCCESS; } /* data is gone (from the pml's perspective). frag callback/release will happen later */ return 1; }
static inline void mca_btl_vader_progress_sends (void) { opal_list_t *list = &mca_btl_vader_component.active_sends; opal_list_item_t *item, *next; mca_btl_vader_frag_t *frag; for (item = opal_list_get_first (list) ; item != opal_list_get_end (list) ; ) { frag = (mca_btl_vader_frag_t *) item; next = opal_list_get_next (item); if (OPAL_LIKELY(frag->hdr->complete)) { opal_list_remove_item (&mca_btl_vader_component.active_sends, item); mca_btl_vader_frag_complete (frag); } item = next; } }
int mca_btl_vader_put (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_base_segment_t *src = des->des_local; mca_btl_base_segment_t *dst = des->des_remote; const size_t size = min(dst->seg_len, src->seg_len); struct iovec src_iov = {.iov_base = src->seg_addr.pval, .iov_len = size}; struct iovec dst_iov = {.iov_base = dst->seg_addr.pval, .iov_len = size}; ssize_t ret; ret = process_vm_writev (endpoint->seg_ds.seg_cpid, &src_iov, 1, &dst_iov, 1, 0); if (ret != (ssize_t)size) { opal_output(0, "Wrote %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno); return OPAL_ERROR; } mca_btl_vader_frag_complete (frag); return OPAL_SUCCESS; }
int mca_btl_vader_put (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_base_segment_t *src = des->des_src; mca_btl_base_segment_t *dst = des->des_dst; const size_t size = min(dst->seg_len, src->seg_len); struct iovec src_iov = {.iov_base = src->seg_addr.pval, .iov_len = size}; struct iovec dst_iov = {.iov_base = dst->seg_addr.pval, .iov_len = size}; ssize_t ret; ret = process_vm_writev (endpoint->seg_ds.seg_cpid, &src_iov, 1, &dst_iov, 1, 0); if (ret != size) { fprintf (stderr, "Wrote %d, expected %u\n", ret, size); return OMPI_ERROR; } mca_btl_vader_frag_complete (frag); return OMPI_SUCCESS; }
/** * Initiate an synchronous get. * * @param btl (IN) BTL module * @param endpoint (IN) BTL addressing information * @param descriptor (IN) Description of the data to be transferred */ int mca_btl_vader_get (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_base_segment_t *src = des->des_src; mca_btl_base_segment_t *dst = des->des_dst; const size_t size = min(dst->seg_len, src->seg_len); mca_mpool_base_registration_t *reg; void *rem_ptr; reg = vader_get_registation (endpoint, src->seg_addr.pval, src->seg_len, 0, &rem_ptr); if (OPAL_UNLIKELY(NULL == rem_ptr)) { return OMPI_ERROR; } vader_memmove (dst->seg_addr.pval, rem_ptr, size); vader_return_registration (reg, endpoint); mca_btl_vader_frag_complete (frag); return OMPI_SUCCESS; }
int mca_btl_vader_put_cma (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_base_segment_t *src = des->des_src; mca_btl_base_segment_t *dst = des->des_dst; const size_t size = min(dst->seg_len, src->seg_len); struct iovec src_iov = {.iov_base = src->seg_addr.pval, .iov_len = size}; struct iovec dst_iov = {.iov_base = dst->seg_addr.pval, .iov_len = size}; ssize_t ret; ret = process_vm_writev (endpoint->segment_data.other.seg_ds->seg_cpid, &src_iov, 1, &dst_iov, 1, 0); if (ret != (ssize_t)size) { opal_output(0, "Wrote %ld, expected %lu, errno = %d\n", (long)ret, (unsigned long)size, errno); return OMPI_ERROR; } /* always call the callback function */ frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; frag->endpoint = endpoint; mca_btl_vader_frag_complete (frag); return OMPI_SUCCESS; } #endif #if OMPI_BTL_VADER_HAVE_KNEM int mca_btl_vader_put_knem (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, struct mca_btl_base_descriptor_t *des) { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des; mca_btl_vader_segment_t *src = (mca_btl_vader_segment_t *) des->des_src; mca_btl_vader_segment_t *dst = (mca_btl_vader_segment_t *) des->des_dst; const size_t size = min(dst->base.seg_len, src->base.seg_len); struct knem_cmd_param_iovec send_iovec; struct knem_cmd_inline_copy icopy; /* Fill in the ioctl data fields. There's no async completion, so we don't need to worry about getting a slot, etc. */ send_iovec.base = (uintptr_t) dst->base.seg_addr.lval; send_iovec.len = size; icopy.local_iovec_array = (uintptr_t) &send_iovec; icopy.local_iovec_nr = 1; icopy.remote_cookie = dst->cookie; icopy.remote_offset = 0; icopy.write = 1; icopy.flags = 0; /* Use the DMA flag if knem supports it *and* the segment length * is greater than the cutoff. Not that if DMA is not supported * or the user specified 0 for knem_dma_min the knem_dma_min was * set to UINT_MAX in mca_btl_vader_knem_init. */ if (mca_btl_vader_component.knem_dma_min <= dst->base.seg_len) { icopy.flags = KNEM_FLAG_DMA; } /* synchronous flags only, no need to specify icopy.async_status_index */ /* When the ioctl returns, the transfer is done and we can invoke the btl callback and return the frag */ if (OPAL_UNLIKELY(0 != ioctl (mca_btl_vader.knem_fd, KNEM_CMD_INLINE_COPY, &icopy))) { return OMPI_ERROR; } if (KNEM_STATUS_FAILED == icopy.current_status) { return OMPI_ERROR; } /* always call the callback function */ frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; frag->endpoint = endpoint; mca_btl_vader_frag_complete (frag); return OMPI_SUCCESS; }