int mca_btl_vader_get_xpmem (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{
mca_rcache_base_registration_t *reg;
void *rem_ptr;
/* silence warning about unused arguments */
(void) local_handle;
(void) remote_handle;
reg = vader_get_registation (endpoint, (void *)(intptr_t) remote_address, size, 0, &rem_ptr);
if (OPAL_UNLIKELY(NULL == rem_ptr)) {
return OPAL_ERROR;
}
vader_memmove (local_address, rem_ptr, size);
vader_return_registration (reg, endpoint);
/* always call the callback function */
cbfunc (btl, endpoint, local_address, local_handle, cbcontext, cbdata, OPAL_SUCCESS);
return OPAL_SUCCESS;
}
int mca_btl_vader_put (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
struct mca_btl_base_descriptor_t *des)
{
mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des;
mca_btl_base_segment_t *src = des->des_local;
mca_btl_base_segment_t *dst = des->des_remote;
const size_t size = min(dst->seg_len, src->seg_len);
mca_mpool_base_registration_t *reg;
void *rem_ptr;
reg = vader_get_registation (endpoint, dst->seg_addr.pval, dst->seg_len, 0, &rem_ptr);
if (OPAL_UNLIKELY(NULL == reg)) {
return OPAL_ERROR;
}
vader_memmove (rem_ptr, src->seg_addr.pval, size);
vader_return_registration (reg, endpoint);
/* always call the callback function */
frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
mca_btl_vader_frag_complete (frag);
return OPAL_SUCCESS;
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_proc_t *proc, int remote_rank) {
mca_btl_vader_component_t *component = &mca_btl_vader_component;
union vader_modex_t *modex;
size_t msg_size;
int rc;
OBJ_CONSTRUCT(ep, mca_btl_vader_endpoint_t);
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
OPAL_MODEX_RECV(rc, &component->super.btl_version,
&proc->proc_name, (void **) &modex, &msg_size);
if (OPAL_SUCCESS != rc) {
return rc;
}
/* attatch to the remote segment */
#if OPAL_BTL_VADER_HAVE_XPMEM
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
/* always use xpmem if it is available */
ep->segment_data.xpmem.apid = xpmem_get (modex->xpmem.seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->segment_data.xpmem.rcache = mca_rcache_base_module_create("vma");
(void) vader_get_registation (ep, modex->xpmem.segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
} else {
#endif
/* store a copy of the segment information for detach */
ep->segment_data.other.seg_ds = malloc (msg_size);
if (NULL == ep->segment_data.other.seg_ds) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
memcpy (ep->segment_data.other.seg_ds, &modex->seg_ds, msg_size);
ep->segment_base = opal_shmem_segment_attach (ep->segment_data.other.seg_ds);
if (NULL == ep->segment_base) {
return OPAL_ERROR;
}
#if OPAL_BTL_VADER_HAVE_XPMEM
}
#endif
OBJ_CONSTRUCT(&ep->lock, opal_mutex_t);
free (modex);
} else {
/* set up the segment base so we can calculate a virtual to real for local pointers */
ep->segment_base = component->my_segment;
}
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
return OPAL_SUCCESS;
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct ompi_proc_t *proc, int remote_rank) {
const int fbox_in_offset = MCA_BTL_VADER_LOCAL_RANK - (MCA_BTL_VADER_LOCAL_RANK > remote_rank);
const int fbox_out_offset = remote_rank - (MCA_BTL_VADER_LOCAL_RANK < remote_rank);
mca_btl_vader_component_t *component = &mca_btl_vader_component;
struct vader_modex_t *modex;
size_t msg_size;
int rc;
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
if (OMPI_SUCCESS != (rc = ompi_modex_recv(&component->super.btl_version,
proc, (void *)&modex, &msg_size))) {
return rc;
}
/* attatch to the remote segment */
#if OMPI_BTL_VADER_HAVE_XPMEM
/* always use xpmem if it is available */
ep->apid = xpmem_get (modex->seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->rcache = mca_rcache_base_module_create("vma");
(void) vader_get_registation (ep, modex->segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
#else
int offset = offsetof (opal_shmem_ds_t, seg_name);
memcpy (&ep->seg_ds, modex->buffer, offset);
memcpy (&ep->seg_ds.seg_base_addr, modex->buffer + offset, sizeof (ep->seg_ds.seg_base_addr));
offset += sizeof (ep->seg_ds.seg_base_addr);
strncpy (ep->seg_ds.seg_name, modex->buffer + offset, OPAL_PATH_MAX);
ep->segment_base = opal_shmem_segment_attach (&ep->seg_ds);
if (NULL == ep->segment_base) {
return rc;
}
#endif
free (modex);
ep->next_fbox_out = 0;
ep->next_fbox_in = 0;
ep->next_sequence = 0;
ep->expected_sequence = 0;
ep->fbox_in = (struct mca_btl_vader_fbox_t * restrict) (ep->segment_base + MCA_BTL_VADER_FIFO_SIZE +
fbox_in_offset * MCA_BTL_VADER_FBOX_PEER_SIZE);
ep->fbox_out = (struct mca_btl_vader_fbox_t * restrict) (component->my_segment + MCA_BTL_VADER_FIFO_SIZE +
fbox_out_offset * MCA_BTL_VADER_FBOX_PEER_SIZE);
} else {
static int mca_btl_vader_component_progress (void)
{
int my_smp_rank = mca_btl_vader_component.my_smp_rank;
vader_fifo_t *fifo = mca_btl_vader_component.fifo[my_smp_rank];
mca_btl_active_message_callback_t *reg;
mca_btl_vader_frag_t frag;
mca_btl_vader_hdr_t *hdr;
mca_mpool_base_registration_t *xpmem_reg = NULL;
/* check active sends for completion */
mca_btl_vader_progress_sends ();
/* check for messages in fast boxes */
mca_btl_vader_check_fboxes ();
/* poll the fifo once */
hdr = vader_fifo_read (fifo);
if (NULL == hdr) {
return 0;
}
reg = mca_btl_base_active_message_trigger + hdr->tag;
frag.base.des_dst = frag.segments;
frag.segments[0].seg_addr.pval = (void *) (hdr + 1);
frag.segments[0].seg_len = hdr->len;
if (OPAL_UNLIKELY(hdr->flags & MCA_BTL_VADER_FLAG_SINGLE_COPY)) {
xpmem_reg = vader_get_registation (hdr->my_smp_rank, hdr->sc_iov.iov_base,
hdr->sc_iov.iov_len, 0);
frag.segments[1].seg_addr.pval = vader_reg_to_ptr (xpmem_reg, hdr->sc_iov.iov_base);
frag.segments[1].seg_len = hdr->sc_iov.iov_len;
/* recv upcall */
frag.base.des_dst_cnt = 2;
reg->cbfunc(&mca_btl_vader.super, hdr->tag, &(frag.base), reg->cbdata);
vader_return_registration (xpmem_reg, hdr->my_smp_rank);
} else {
frag.base.des_dst_cnt = 1;
reg->cbfunc(&mca_btl_vader.super, hdr->tag, &(frag.base), reg->cbdata);
}
/* return the fragment */
hdr->complete = true;
return 1;
}
/**
* Initiate an synchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
int mca_btl_vader_get (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
struct mca_btl_base_descriptor_t *des)
{
mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) des;
mca_btl_base_segment_t *src = des->des_src;
mca_btl_base_segment_t *dst = des->des_dst;
const size_t size = min(dst->seg_len, src->seg_len);
mca_mpool_base_registration_t *reg;
void *rem_ptr;
reg = vader_get_registation (endpoint, src->seg_addr.pval, src->seg_len, 0, &rem_ptr);
if (OPAL_UNLIKELY(NULL == rem_ptr)) {
return OMPI_ERROR;
}
vader_memmove (dst->seg_addr.pval, rem_ptr, size);
vader_return_registration (reg, endpoint);
mca_btl_vader_frag_complete (frag);
return OMPI_SUCCESS;
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct ompi_proc_t *proc, int remote_rank) {
const int fbox_in_offset = MCA_BTL_VADER_LOCAL_RANK - (MCA_BTL_VADER_LOCAL_RANK > remote_rank);
const int fbox_out_offset = remote_rank - (MCA_BTL_VADER_LOCAL_RANK < remote_rank);
mca_btl_vader_component_t *component = &mca_btl_vader_component;
struct vader_modex_t *modex;
size_t msg_size;
int rc;
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
if (OMPI_SUCCESS != (rc = ompi_modex_recv(&component->super.btl_version,
proc, (void *)&modex, &msg_size))) {
return rc;
}
ep->apid = xpmem_get (modex->seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->rcache = mca_rcache_base_module_create("vma");
ep->next_fbox_out = 0;
ep->next_fbox_in = 0;
/* attatch to the remote segment */
(void) vader_get_registation (ep, modex->segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
ep->fbox_in = ep->segment_base + 4096 + fbox_in_offset * MCA_BTL_VADER_FBOX_PEER_SIZE;
ep->fbox_out = component->my_segment + 4096 + fbox_out_offset * MCA_BTL_VADER_FBOX_PEER_SIZE;
} else {
/* set up the segment base so we can calculate a virtual to real for local pointers */
ep->segment_base = component->my_segment;
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
}
return OMPI_SUCCESS;
}
