/* ////////////////////////////////////////////////////////////////////////// */
static mca_common_sm_module_t *
attach_and_init(opal_shmem_ds_t *shmem_bufp,
size_t size,
size_t size_ctl_structure,
size_t data_seg_alignment,
bool first_call)
{
mca_common_sm_module_t *map = NULL;
mca_common_sm_seg_header_t *seg = NULL;
unsigned char *addr = NULL;
/* attach to the specified segment. note that at this point, the contents of
* *shmem_bufp have already been initialized via opal_shmem_segment_create.
*/
if (NULL == (seg = (mca_common_sm_seg_header_t *)
opal_shmem_segment_attach(shmem_bufp))) {
return NULL;
}
opal_atomic_rmb();
if (NULL == (map = OBJ_NEW(mca_common_sm_module_t))) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
(void)opal_shmem_segment_detach(shmem_bufp);
return NULL;
}
/* copy meta information into common sm module
* from ====> to */
if (OPAL_SUCCESS != opal_shmem_ds_copy(shmem_bufp, &map->shmem_ds)) {
(void)opal_shmem_segment_detach(shmem_bufp);
free(map);
return NULL;
}
/* the first entry in the file is the control structure. the first
* entry in the control structure is an mca_common_sm_seg_header_t
* element.
*/
map->module_seg = seg;
addr = ((unsigned char *)seg) + size_ctl_structure;
/* if we have a data segment (i.e., if 0 != data_seg_alignment),
* then make it the first aligned address after the control
* structure. IF THIS HAPPENS, THIS IS A PROGRAMMING ERROR IN
* OPEN MPI!
*/
if (0 != data_seg_alignment) {
addr = OPAL_ALIGN_PTR(addr, data_seg_alignment, unsigned char *);
/* is addr past end of the shared memory segment? */
if ((unsigned char *)seg + shmem_bufp->seg_size < addr) {
opal_show_help("help-mpi-common-sm.txt", "mmap too small", 1,
opal_proc_local_get()->proc_hostname,
(unsigned long)shmem_bufp->seg_size,
(unsigned long)size_ctl_structure,
(unsigned long)data_seg_alignment);
(void)opal_shmem_segment_detach(shmem_bufp);
free(map);
return NULL;
}
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_proc_t *proc, int remote_rank) {
mca_btl_vader_component_t *component = &mca_btl_vader_component;
union vader_modex_t *modex;
size_t msg_size;
int rc;
OBJ_CONSTRUCT(ep, mca_btl_vader_endpoint_t);
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
OPAL_MODEX_RECV(rc, &component->super.btl_version,
&proc->proc_name, (void **) &modex, &msg_size);
if (OPAL_SUCCESS != rc) {
return rc;
}
/* attatch to the remote segment */
#if OPAL_BTL_VADER_HAVE_XPMEM
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
/* always use xpmem if it is available */
ep->segment_data.xpmem.apid = xpmem_get (modex->xpmem.seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->segment_data.xpmem.rcache = mca_rcache_base_module_create("vma");
(void) vader_get_registation (ep, modex->xpmem.segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
} else {
#endif
/* store a copy of the segment information for detach */
ep->segment_data.other.seg_ds = malloc (msg_size);
if (NULL == ep->segment_data.other.seg_ds) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
memcpy (ep->segment_data.other.seg_ds, &modex->seg_ds, msg_size);
ep->segment_base = opal_shmem_segment_attach (ep->segment_data.other.seg_ds);
if (NULL == ep->segment_base) {
return OPAL_ERROR;
}
#if OPAL_BTL_VADER_HAVE_XPMEM
}
#endif
OBJ_CONSTRUCT(&ep->lock, opal_mutex_t);
free (modex);
} else {
/* set up the segment base so we can calculate a virtual to real for local pointers */
ep->segment_base = component->my_segment;
}
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
return OPAL_SUCCESS;
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct ompi_proc_t *proc, int remote_rank) {
const int fbox_in_offset = MCA_BTL_VADER_LOCAL_RANK - (MCA_BTL_VADER_LOCAL_RANK > remote_rank);
const int fbox_out_offset = remote_rank - (MCA_BTL_VADER_LOCAL_RANK < remote_rank);
mca_btl_vader_component_t *component = &mca_btl_vader_component;
struct vader_modex_t *modex;
size_t msg_size;
int rc;
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
if (OMPI_SUCCESS != (rc = ompi_modex_recv(&component->super.btl_version,
proc, (void *)&modex, &msg_size))) {
return rc;
}
/* attatch to the remote segment */
#if OMPI_BTL_VADER_HAVE_XPMEM
/* always use xpmem if it is available */
ep->apid = xpmem_get (modex->seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->rcache = mca_rcache_base_module_create("vma");
(void) vader_get_registation (ep, modex->segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
#else
int offset = offsetof (opal_shmem_ds_t, seg_name);
memcpy (&ep->seg_ds, modex->buffer, offset);
memcpy (&ep->seg_ds.seg_base_addr, modex->buffer + offset, sizeof (ep->seg_ds.seg_base_addr));
offset += sizeof (ep->seg_ds.seg_base_addr);
strncpy (ep->seg_ds.seg_name, modex->buffer + offset, OPAL_PATH_MAX);
ep->segment_base = opal_shmem_segment_attach (&ep->seg_ds);
if (NULL == ep->segment_base) {
return rc;
}
#endif
free (modex);
ep->next_fbox_out = 0;
ep->next_fbox_in = 0;
ep->next_sequence = 0;
ep->expected_sequence = 0;
ep->fbox_in = (struct mca_btl_vader_fbox_t * restrict) (ep->segment_base + MCA_BTL_VADER_FIFO_SIZE +
fbox_in_offset * MCA_BTL_VADER_FBOX_PEER_SIZE);
ep->fbox_out = (struct mca_btl_vader_fbox_t * restrict) (component->my_segment + MCA_BTL_VADER_FIFO_SIZE +
fbox_out_offset * MCA_BTL_VADER_FBOX_PEER_SIZE);
} else {
