static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct opal_proc_t *proc, int remote_rank) {
mca_btl_vader_component_t *component = &mca_btl_vader_component;
union vader_modex_t *modex;
size_t msg_size;
int rc;
OBJ_CONSTRUCT(ep, mca_btl_vader_endpoint_t);
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
OPAL_MODEX_RECV(rc, &component->super.btl_version,
&proc->proc_name, (void **) &modex, &msg_size);
if (OPAL_SUCCESS != rc) {
return rc;
}
/* attatch to the remote segment */
#if OPAL_BTL_VADER_HAVE_XPMEM
if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) {
/* always use xpmem if it is available */
ep->segment_data.xpmem.apid = xpmem_get (modex->xpmem.seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->segment_data.xpmem.rcache = mca_rcache_base_module_create("vma");
(void) vader_get_registation (ep, modex->xpmem.segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
} else {
#endif
/* store a copy of the segment information for detach */
ep->segment_data.other.seg_ds = malloc (msg_size);
if (NULL == ep->segment_data.other.seg_ds) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
memcpy (ep->segment_data.other.seg_ds, &modex->seg_ds, msg_size);
ep->segment_base = opal_shmem_segment_attach (ep->segment_data.other.seg_ds);
if (NULL == ep->segment_base) {
return OPAL_ERROR;
}
#if OPAL_BTL_VADER_HAVE_XPMEM
}
#endif
OBJ_CONSTRUCT(&ep->lock, opal_mutex_t);
free (modex);
} else {
/* set up the segment base so we can calculate a virtual to real for local pointers */
ep->segment_base = component->my_segment;
}
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
return OPAL_SUCCESS;
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct ompi_proc_t *proc, int remote_rank) {
const int fbox_in_offset = MCA_BTL_VADER_LOCAL_RANK - (MCA_BTL_VADER_LOCAL_RANK > remote_rank);
const int fbox_out_offset = remote_rank - (MCA_BTL_VADER_LOCAL_RANK < remote_rank);
mca_btl_vader_component_t *component = &mca_btl_vader_component;
struct vader_modex_t *modex;
size_t msg_size;
int rc;
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
if (OMPI_SUCCESS != (rc = ompi_modex_recv(&component->super.btl_version,
proc, (void *)&modex, &msg_size))) {
return rc;
}
/* attatch to the remote segment */
#if OMPI_BTL_VADER_HAVE_XPMEM
/* always use xpmem if it is available */
ep->apid = xpmem_get (modex->seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->rcache = mca_rcache_base_module_create("vma");
(void) vader_get_registation (ep, modex->segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
#else
int offset = offsetof (opal_shmem_ds_t, seg_name);
memcpy (&ep->seg_ds, modex->buffer, offset);
memcpy (&ep->seg_ds.seg_base_addr, modex->buffer + offset, sizeof (ep->seg_ds.seg_base_addr));
offset += sizeof (ep->seg_ds.seg_base_addr);
strncpy (ep->seg_ds.seg_name, modex->buffer + offset, OPAL_PATH_MAX);
ep->segment_base = opal_shmem_segment_attach (&ep->seg_ds);
if (NULL == ep->segment_base) {
return rc;
}
#endif
free (modex);
ep->next_fbox_out = 0;
ep->next_fbox_in = 0;
ep->next_sequence = 0;
ep->expected_sequence = 0;
ep->fbox_in = (struct mca_btl_vader_fbox_t * restrict) (ep->segment_base + MCA_BTL_VADER_FIFO_SIZE +
fbox_in_offset * MCA_BTL_VADER_FBOX_PEER_SIZE);
ep->fbox_out = (struct mca_btl_vader_fbox_t * restrict) (component->my_segment + MCA_BTL_VADER_FIFO_SIZE +
fbox_out_offset * MCA_BTL_VADER_FBOX_PEER_SIZE);
} else {
/**
* test_fork - test if forks are handled properly
* Description:
* Called by xpmem_master, but do nothing. xpmem_proc1 does the fork.
* Return Values:
* Success: 0
* Failure: -2
*/
int test_fork(test_args *xpmem_args)
{
xpmem_segid_t segid;
xpmem_apid_t apid;
struct xpmem_addr addr;
int i, ret=0, *data;
segid = strtol(xpmem_args->share, NULL, 16);
apid = xpmem_get(segid, XPMEM_RDWR, XPMEM_PERMIT_MODE, NULL);
addr.apid = apid;
addr.offset = PAGE_SIZE;
data = (int *)xpmem_attach(addr, PAGE_SIZE, NULL);
if (data == (void *)-1) {
perror("xpmem_attach");
return -2;
}
printf("xpmem_proc2: mypid = %d\n", getpid());
printf("xpmem_proc2: segid = %llx\n", segid);
printf("xpmem_proc2: attached at %p\n", data);
printf("xpmem_proc2: reading to pin pages\n");
for (i = 0; i < PAGE_INT_SIZE; i++) {
if (*(data + i) != PAGE_INT_SIZE + i) {
printf("xpmem_proc2: ***mismatch at %d: expected %lu "
"got %d\n", i, PAGE_INT_SIZE + i, *(data + i));
ret = -2;
}
}
/* Now wait for xpmem_proc1 to invoke COW */
printf("xpmem_proc2: waiting for COW...\n\n");
while (xpmem_args->share[COW_LOCK_INDEX] == 0) {
xpmem_args->share[COW_LOCK_INDEX] = 1;
}
sleep(1);
printf("xpmem_proc2: adding 1 to all elems\n\n");
for (i = 0; i < PAGE_INT_SIZE; i++)
*(data + i) += 1;
xpmem_detach(data);
xpmem_release(apid);
return ret;
}
static ucs_status_t uct_xpmem_attach(uct_mm_id_t mmid, size_t length,
void *remote_address, void **local_address,
uint64_t *cookie, const char *path)
{
struct xpmem_addr addr;
ucs_status_t status;
ptrdiff_t offset;
void *address;
addr.offset = 0;
addr.apid = xpmem_get(mmid, XPMEM_RDWR, XPMEM_PERMIT_MODE, NULL);
VALGRIND_MAKE_MEM_DEFINED(&addr.apid, sizeof(addr.apid));
if (addr.apid < 0) {
ucs_error("Failed to acquire xpmem segment 0x%"PRIx64": %m", mmid);
status = UCS_ERR_IO_ERROR;
goto err_xget;
}
ucs_trace("xpmem acquired segment 0x%"PRIx64" apid 0x%llx remote_address %p",
mmid, addr.apid, remote_address);
offset = ((uintptr_t)remote_address) % ucs_get_page_size();
address = xpmem_attach(addr, length + offset, NULL);
VALGRIND_MAKE_MEM_DEFINED(&address, sizeof(address));
if (address == MAP_FAILED) {
ucs_error("Failed to attach xpmem segment 0x%"PRIx64" apid 0x%llx "
"with length %zu: %m", mmid, addr.apid, length);
status = UCS_ERR_IO_ERROR;
goto err_xattach;
}
VALGRIND_MAKE_MEM_DEFINED(address + offset, length);
*local_address = address + offset;
*cookie = addr.apid;
ucs_trace("xpmem attached segment 0x%"PRIx64" apid 0x%llx %p..%p at %p (+%zd)",
mmid, addr.apid, remote_address, remote_address + length, address, offset);
return UCS_OK;
err_xattach:
xpmem_release(addr.apid);
err_xget:
return status;
}
static int init_vader_endpoint (struct mca_btl_base_endpoint_t *ep, struct ompi_proc_t *proc, int remote_rank) {
const int fbox_in_offset = MCA_BTL_VADER_LOCAL_RANK - (MCA_BTL_VADER_LOCAL_RANK > remote_rank);
const int fbox_out_offset = remote_rank - (MCA_BTL_VADER_LOCAL_RANK < remote_rank);
mca_btl_vader_component_t *component = &mca_btl_vader_component;
struct vader_modex_t *modex;
size_t msg_size;
int rc;
ep->peer_smp_rank = remote_rank;
if (remote_rank != MCA_BTL_VADER_LOCAL_RANK) {
if (OMPI_SUCCESS != (rc = ompi_modex_recv(&component->super.btl_version,
proc, (void *)&modex, &msg_size))) {
return rc;
}
ep->apid = xpmem_get (modex->seg_id, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void *) 0666);
ep->rcache = mca_rcache_base_module_create("vma");
ep->next_fbox_out = 0;
ep->next_fbox_in = 0;
/* attatch to the remote segment */
(void) vader_get_registation (ep, modex->segment_base, mca_btl_vader_component.segment_size,
MCA_MPOOL_FLAGS_PERSIST, (void **) &ep->segment_base);
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
ep->fbox_in = ep->segment_base + 4096 + fbox_in_offset * MCA_BTL_VADER_FBOX_PEER_SIZE;
ep->fbox_out = component->my_segment + 4096 + fbox_out_offset * MCA_BTL_VADER_FBOX_PEER_SIZE;
} else {
/* set up the segment base so we can calculate a virtual to real for local pointers */
ep->segment_base = component->my_segment;
ep->fifo = (struct vader_fifo_t *) ep->segment_base;
}
return OMPI_SUCCESS;
}
int
shmem_transport_xpmem_startup(void)
{
int ret, i, peer_num, num_on_node = 0;
struct share_info_t info;
struct xpmem_addr addr;
for (i = 0 ; i < shmem_internal_num_pes; ++i) {
if (-1 != SHMEM_GET_RANK_SAME_NODE(i)) {
num_on_node++;
}
}
/* allocate space for local peers */
shmem_transport_xpmem_peers = calloc(num_on_node,
sizeof(struct shmem_transport_xpmem_peer_info_t));
if (NULL == shmem_transport_xpmem_peers) return 1;
/* get local peer info and map into our address space ... */
for (i = 0 ; i < shmem_internal_num_pes; ++i) {
peer_num = SHMEM_GET_RANK_SAME_NODE(i);
if (-1 == peer_num) continue;
if (shmem_internal_my_pe == i) {
shmem_transport_xpmem_peers[peer_num].data_ptr =
shmem_internal_data_base;
shmem_transport_xpmem_peers[peer_num].heap_ptr =
shmem_internal_heap_base;
} else {
ret = shmem_runtime_get(i, "xpmem-segids", &info, sizeof(struct share_info_t));
if (0 != ret) {
fprintf(stderr, "[%03d] ERROR: runtime_get failed: %d\n",
shmem_internal_my_pe, ret);
return 1;
}
shmem_transport_xpmem_peers[peer_num].data_apid =
xpmem_get(info.data_seg, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void*)0666);
if (shmem_transport_xpmem_peers[peer_num].data_apid < 0) {
fprintf(stderr, "[%03d] ERROR: could not get data apid: %s\n",
shmem_internal_my_pe, strerror(errno));
return 1;
}
addr.apid = shmem_transport_xpmem_peers[peer_num].data_apid;
addr.offset = 0;
shmem_transport_xpmem_peers[peer_num].data_attach_ptr =
xpmem_attach(addr, info.data_len, NULL);
if ((size_t) shmem_transport_xpmem_peers[peer_num].data_ptr == XPMEM_MAXADDR_SIZE) {
fprintf(stderr, "[%03d] ERROR: could not get data segment: %s\n",
shmem_internal_my_pe, strerror(errno));
return 1;
}
shmem_transport_xpmem_peers[peer_num].data_ptr =
(char*) shmem_transport_xpmem_peers[peer_num].data_attach_ptr + info.data_off;
shmem_transport_xpmem_peers[peer_num].heap_apid =
xpmem_get(info.heap_seg, XPMEM_RDWR, XPMEM_PERMIT_MODE, (void*)0666);
if (shmem_transport_xpmem_peers[peer_num].heap_apid < 0) {
fprintf(stderr, "[%03d] ERROR: could not get heap apid: %s\n",
shmem_internal_my_pe, strerror(errno));
return 1;
}
addr.apid = shmem_transport_xpmem_peers[peer_num].heap_apid;
addr.offset = 0;
shmem_transport_xpmem_peers[peer_num].heap_attach_ptr =
xpmem_attach(addr, info.heap_len, NULL);
if ((size_t) shmem_transport_xpmem_peers[peer_num].heap_ptr == XPMEM_MAXADDR_SIZE) {
fprintf(stderr, "[%03d] ERROR: could not get data segment: %s\n",
shmem_internal_my_pe, strerror(errno));
return 1;
}
shmem_transport_xpmem_peers[peer_num].heap_ptr =
(char*) shmem_transport_xpmem_peers[peer_num].heap_attach_ptr + info.heap_off;
}
}
return 0;
}
/*
* User ioctl to the XPMEM driver. Only 64-bit user applications are
* supported.
*/
static long
xpmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
long ret;
switch (cmd) {
case XPMEM_CMD_VERSION: {
return XPMEM_CURRENT_VERSION;
}
case XPMEM_CMD_MAKE: {
struct xpmem_cmd_make make_info;
xpmem_segid_t segid;
if (copy_from_user(&make_info, (void __user *)arg,
sizeof(struct xpmem_cmd_make)))
return -EFAULT;
ret = xpmem_make(make_info.vaddr, make_info.size,
make_info.permit_type,
(void *)make_info.permit_value, &segid);
if (ret != 0)
return ret;
if (put_user(segid,
&((struct xpmem_cmd_make __user *)arg)->segid)) {
(void)xpmem_remove(segid);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_REMOVE: {
struct xpmem_cmd_remove remove_info;
if (copy_from_user(&remove_info, (void __user *)arg,
sizeof(struct xpmem_cmd_remove)))
return -EFAULT;
return xpmem_remove(remove_info.segid);
}
case XPMEM_CMD_GET: {
struct xpmem_cmd_get get_info;
xpmem_apid_t apid;
if (copy_from_user(&get_info, (void __user *)arg,
sizeof(struct xpmem_cmd_get)))
return -EFAULT;
ret = xpmem_get(get_info.segid, get_info.flags,
get_info.permit_type,
(void *)get_info.permit_value, &apid);
if (ret != 0)
return ret;
if (put_user(apid,
&((struct xpmem_cmd_get __user *)arg)->apid)) {
(void)xpmem_release(apid);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_RELEASE: {
struct xpmem_cmd_release release_info;
if (copy_from_user(&release_info, (void __user *)arg,
sizeof(struct xpmem_cmd_release)))
return -EFAULT;
return xpmem_release(release_info.apid);
}
case XPMEM_CMD_ATTACH: {
struct xpmem_cmd_attach attach_info;
u64 at_vaddr;
if (copy_from_user(&attach_info, (void __user *)arg,
sizeof(struct xpmem_cmd_attach)))
return -EFAULT;
ret = xpmem_attach(file, attach_info.apid, attach_info.offset,
attach_info.size, attach_info.vaddr,
attach_info.fd, attach_info.flags,
&at_vaddr);
if (ret != 0)
return ret;
if (put_user(at_vaddr,
&((struct xpmem_cmd_attach __user *)arg)->vaddr)) {
(void)xpmem_detach(at_vaddr);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_DETACH: {
struct xpmem_cmd_detach detach_info;
if (copy_from_user(&detach_info, (void __user *)arg,
sizeof(struct xpmem_cmd_detach)))
return -EFAULT;
return xpmem_detach(detach_info.vaddr);
}
case XPMEM_CMD_FORK_BEGIN: {
return xpmem_fork_begin();
}
case XPMEM_CMD_FORK_END: {
return xpmem_fork_end();
}
default:
break;
}
return -ENOIOCTLCMD;
}