/**
* test_two_shares - same as test_base, but with two sources and two consumers
* Description:
* Two consumers of two different apids.
* Return Values:
* Success: 0
* Failure: -2
*/
int test_two_shares(test_args *xpmem_args)
{
xpmem_segid_t segid[2];
xpmem_apid_t apid[2];
int i, ret=0, *data[2];
char *tmp;
tmp = xpmem_args->share;
segid[0] = strtol(tmp, NULL, 16);
tmp += strlen(tmp) + 1;
segid[1] = strtol(tmp, NULL, 16);
data[0] = attach_segid(segid[0], &apid[0]);
data[1] = attach_segid(segid[1], &apid[1]);
if (data[0] == (void *)-1 || data[1] == (void *)-1) {
perror("xpmem_attach");
return -2;
}
printf("xpmem_proc2: mypid = %d\n", getpid());
printf("xpmem_proc2: segid[0] = %llx\n", segid[0]);
printf("xpmem_proc2: segid[1] = %llx\n", segid[1]);
printf("xpmem_proc2: data[0] attached at %p\n", data[0]);
printf("xpmem_proc2: data[1] attached at %p\n", data[1]);
printf("xpmem_proc2: adding 1 to all elems using %p\n", data[0]);
printf("xpmem_proc2: adding 1 to all elems using %p\n\n", data[1]);
for (i = 0; i < SHARE_INT_SIZE; i++) {
if (*(data[0] + i) != i) {
printf("xpmem_proc2: ***mismatch at %d: expected %d "
"got %d\n", i, i, *(data[0] + i));
ret = -2;
}
if (*(data[1] + i) != i) {
printf("xpmem_proc2: ***mismatch at %d: expected %d "
"got %d\n", i, i, *(data[1] + i));
ret = -2;
}
*(data[0] + i) += 1;
*(data[1] + i) += 1;
}
xpmem_detach(data[0]);
xpmem_detach(data[1]);
xpmem_release(apid[0]);
xpmem_release(apid[1]);
return ret;
}
/**
* test_base - a simple test to share and attach
* Description:
* Creates a share (initialized to a random value), calls a second process
* to attach to the shared address and increment its value.
* Return Values:
* Success: 0
* Failure: -2
*/
int test_base(test_args *xpmem_args)
{
xpmem_segid_t segid;
xpmem_apid_t apid;
int i, ret=0, *data;
segid = strtol(xpmem_args->share, NULL, 16);
data = attach_segid(segid, &apid);
if (data == (void *)-1) {
perror("xpmem_attach");
return -2;
}
printf("xpmem_proc2: mypid = %d\n", getpid());
printf("xpmem_proc2: segid = %llx\n", segid);
printf("xpmem_proc2: attached at %p\n", data);
printf("xpmem_proc2: adding 1 to all elems\n\n");
for (i = 0; i < SHARE_INT_SIZE; i++) {
if (*(data + i) != i) {
printf("xpmem_proc2: ***mismatch at %d: expected %d "
"got %d\n", i, i, *(data + i));
ret = -2;
}
*(data + i) += 1;
}
xpmem_detach(data);
xpmem_release(apid);
return ret;
}
static ucs_status_t uct_xpmem_detach(uct_mm_remote_seg_t *mm_desc)
{
xpmem_apid_t apid = mm_desc->cookie;
void *address;
int ret;
address = ucs_align_down_pow2_ptr(mm_desc->address, ucs_get_page_size());
ucs_trace("xpmem detaching address %p", address);
ret = xpmem_detach(address);
if (ret < 0) {
ucs_error("Failed to xpmem_detach: %m");
return UCS_ERR_IO_ERROR;
}
VALGRIND_MAKE_MEM_UNDEFINED(mm_desc->address, mm_desc->length);
ucs_trace("xpmem releasing segment apid 0x%llx", apid);
ret = xpmem_release(apid);
if (ret < 0) {
ucs_error("Failed to release xpmem segment apid 0x%llx", apid);
return UCS_ERR_IO_ERROR;
}
return UCS_OK;
}
int
shmem_transport_xpmem_fini(void)
{
int i, peer_num;
if (NULL != shmem_transport_xpmem_peers) {
for (i = 0 ; i < shmem_internal_num_pes; ++i) {
peer_num = SHMEM_GET_RANK_SAME_NODE(i);
if (-1 == peer_num) continue;
if (shmem_internal_my_pe == i) continue;
if (NULL != shmem_transport_xpmem_peers[peer_num].data_ptr) {
xpmem_detach(shmem_transport_xpmem_peers[peer_num].data_attach_ptr);
}
if (0 != shmem_transport_xpmem_peers[peer_num].data_apid) {
xpmem_release(shmem_transport_xpmem_peers[peer_num].data_apid);
}
if (NULL != shmem_transport_xpmem_peers[peer_num].heap_ptr) {
xpmem_detach(shmem_transport_xpmem_peers[peer_num].heap_attach_ptr);
}
if (0 != shmem_transport_xpmem_peers[peer_num].heap_apid) {
xpmem_release(shmem_transport_xpmem_peers[peer_num].heap_apid);
}
}
free(shmem_transport_xpmem_peers);
}
if (0 != my_info.data_seg) {
xpmem_remove(my_info.data_seg);
}
if (0 != my_info.heap_seg) {
xpmem_remove(my_info.heap_seg);
}
return 0;
}
void vader_return_registration (mca_rcache_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
{
mca_rcache_base_vma_module_t *vma_module = mca_btl_vader_component.vma_module;
int32_t ref_count;
ref_count = opal_atomic_add_32 (®->ref_count, -1);
if (OPAL_UNLIKELY(0 == ref_count && !(reg->flags & MCA_RCACHE_FLAGS_PERSIST))) {
/* protect rcache access */
mca_rcache_base_vma_delete (vma_module, reg);
opal_memchecker_base_mem_noaccess (reg->rcache_context, (uintptr_t)(reg->bound - reg->base));
(void)xpmem_detach (reg->rcache_context);
OBJ_RELEASE (reg);
}
}
/**
* test_fork - test if forks are handled properly
* Description:
* Called by xpmem_master, but do nothing. xpmem_proc1 does the fork.
* Return Values:
* Success: 0
* Failure: -2
*/
int test_fork(test_args *xpmem_args)
{
xpmem_segid_t segid;
xpmem_apid_t apid;
struct xpmem_addr addr;
int i, ret=0, *data;
segid = strtol(xpmem_args->share, NULL, 16);
apid = xpmem_get(segid, XPMEM_RDWR, XPMEM_PERMIT_MODE, NULL);
addr.apid = apid;
addr.offset = PAGE_SIZE;
data = (int *)xpmem_attach(addr, PAGE_SIZE, NULL);
if (data == (void *)-1) {
perror("xpmem_attach");
return -2;
}
printf("xpmem_proc2: mypid = %d\n", getpid());
printf("xpmem_proc2: segid = %llx\n", segid);
printf("xpmem_proc2: attached at %p\n", data);
printf("xpmem_proc2: reading to pin pages\n");
for (i = 0; i < PAGE_INT_SIZE; i++) {
if (*(data + i) != PAGE_INT_SIZE + i) {
printf("xpmem_proc2: ***mismatch at %d: expected %lu "
"got %d\n", i, PAGE_INT_SIZE + i, *(data + i));
ret = -2;
}
}
/* Now wait for xpmem_proc1 to invoke COW */
printf("xpmem_proc2: waiting for COW...\n\n");
while (xpmem_args->share[COW_LOCK_INDEX] == 0) {
xpmem_args->share[COW_LOCK_INDEX] = 1;
}
sleep(1);
printf("xpmem_proc2: adding 1 to all elems\n\n");
for (i = 0; i < PAGE_INT_SIZE; i++)
*(data + i) += 1;
xpmem_detach(data);
xpmem_release(apid);
return ret;
}
void vader_return_registration (mca_mpool_base_registration_t *reg, struct mca_btl_base_endpoint_t *ep)
{
struct mca_rcache_base_module_t *rcache = ep->rcache;
int32_t ref_count;
ref_count = opal_atomic_add_32 (®->ref_count, -1);
if (OPAL_UNLIKELY(0 == ref_count && !(reg->flags & MCA_MPOOL_FLAGS_PERSIST))) {
/* protect rcache access */
OPAL_THREAD_LOCK(&ep->lock);
rcache->rcache_delete (rcache, reg);
OPAL_THREAD_UNLOCK(&ep->lock);
opal_memchecker_base_mem_noaccess (reg->alloc_base, (uintptr_t)(reg->bound - reg->base));
(void)xpmem_detach (reg->alloc_base);
OBJ_RELEASE (reg);
}
}
/* look up the remote pointer in the peer rcache and attach if
* necessary */
mca_mpool_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
size_t size, int flags, void **local_ptr)
{
struct mca_rcache_base_module_t *rcache = ep->rcache;
mca_mpool_base_registration_t *regs[10], *reg = NULL;
xpmem_addr_t xpmem_addr;
uintptr_t base, bound;
int rc, i;
/* protect rcache access */
OPAL_THREAD_LOCK(&ep->lock);
/* use btl/self for self communication */
assert (ep->peer_smp_rank != MCA_BTL_VADER_LOCAL_RANK);
base = (uintptr_t) down_align_addr(rem_ptr, mca_btl_vader_component.log_attach_align);
bound = (uintptr_t) up_align_addr((void *)((uintptr_t) rem_ptr + size - 1),
mca_btl_vader_component.log_attach_align) + 1;
if (OPAL_UNLIKELY(bound > VADER_MAX_ADDRESS)) {
bound = VADER_MAX_ADDRESS;
}
/* several segments may match the base pointer */
rc = rcache->rcache_find_all (rcache, (void *) base, bound - base, regs, 10);
for (i = 0 ; i < rc ; ++i) {
if (bound <= (uintptr_t)regs[i]->bound && base >= (uintptr_t)regs[i]->base) {
opal_atomic_add (®s[i]->ref_count, 1);
reg = regs[i];
goto reg_found;
}
if (regs[i]->flags & MCA_MPOOL_FLAGS_PERSIST) {
continue;
}
/* remove this pointer from the rcache and decrement its reference count
(so it is detached later) */
rc = rcache->rcache_delete (rcache, regs[i]);
if (OPAL_UNLIKELY(0 != rc)) {
/* someone beat us to it? */
break;
}
/* start the new segment from the lower of the two bases */
base = (uintptr_t) regs[i]->base < base ? (uintptr_t) regs[i]->base : base;
opal_atomic_add (®s[i]->ref_count, -1);
if (OPAL_LIKELY(0 == regs[i]->ref_count)) {
/* this pointer is not in use */
(void) xpmem_detach (regs[i]->alloc_base);
OBJ_RELEASE(regs[i]);
}
break;
}
reg = OBJ_NEW(mca_mpool_base_registration_t);
if (OPAL_LIKELY(NULL != reg)) {
/* stick around for awhile */
reg->ref_count = 2;
reg->base = (unsigned char *) base;
reg->bound = (unsigned char *) bound;
reg->flags = flags;
#if defined(HAVE_SN_XPMEM_H)
xpmem_addr.id = ep->apid;
#else
xpmem_addr.apid = ep->apid;
#endif
xpmem_addr.offset = base;
reg->alloc_base = xpmem_attach (xpmem_addr, bound - base, NULL);
if (OPAL_UNLIKELY((void *)-1 == reg->alloc_base)) {
OPAL_THREAD_UNLOCK(&ep->lock);
OBJ_RELEASE(reg);
return NULL;
}
opal_memchecker_base_mem_defined (reg->alloc_base, bound - base);
rcache->rcache_insert (rcache, reg, 0);
}
reg_found:
opal_atomic_wmb ();
*local_ptr = (void *) ((uintptr_t) reg->alloc_base +
(ptrdiff_t)((uintptr_t) rem_ptr - (uintptr_t) reg->base));
OPAL_THREAD_UNLOCK(&ep->lock);
return reg;
}
/* look up the remote pointer in the peer rcache and attach if
* necessary */
mca_rcache_base_registration_t *vader_get_registation (struct mca_btl_base_endpoint_t *ep, void *rem_ptr,
size_t size, int flags, void **local_ptr)
{
mca_rcache_base_vma_module_t *vma_module = mca_btl_vader_component.vma_module;
uint64_t attach_align = 1 << mca_btl_vader_component.log_attach_align;
mca_rcache_base_registration_t *reg = NULL;
vader_check_reg_ctx_t check_ctx = {.ep = ep, .reg = ®, .vma_module = vma_module};
xpmem_addr_t xpmem_addr;
uintptr_t base, bound;
int rc;
base = OPAL_DOWN_ALIGN((uintptr_t) rem_ptr, attach_align, uintptr_t);
bound = OPAL_ALIGN((uintptr_t) rem_ptr + size - 1, attach_align, uintptr_t) + 1;
if (OPAL_UNLIKELY(bound > VADER_MAX_ADDRESS)) {
bound = VADER_MAX_ADDRESS;
}
check_ctx.base = base;
check_ctx.bound = bound;
/* several segments may match the base pointer */
rc = mca_rcache_base_vma_iterate (vma_module, (void *) base, bound - base, vader_check_reg, &check_ctx);
if (2 == rc) {
/* start the new segment from the lower of the two bases */
base = (uintptr_t) reg->base < base ? (uintptr_t) reg->base : base;
if (OPAL_LIKELY(0 == opal_atomic_add_32 (®->ref_count, -1))) {
/* this pointer is not in use */
(void) xpmem_detach (reg->rcache_context);
OBJ_RELEASE(reg);
}
reg = NULL;
}
if (NULL == reg) {
reg = OBJ_NEW(mca_rcache_base_registration_t);
if (OPAL_LIKELY(NULL != reg)) {
/* stick around for awhile */
reg->ref_count = 2;
reg->base = (unsigned char *) base;
reg->bound = (unsigned char *) bound;
reg->flags = flags;
reg->alloc_base = (void *) (intptr_t) ep->peer_smp_rank;
#if defined(HAVE_SN_XPMEM_H)
xpmem_addr.id = ep->segment_data.xpmem.apid;
#else
xpmem_addr.apid = ep->segment_data.xpmem.apid;
#endif
xpmem_addr.offset = base;
reg->rcache_context = xpmem_attach (xpmem_addr, bound - base, NULL);
if (OPAL_UNLIKELY((void *)-1 == reg->rcache_context)) {
OBJ_RELEASE(reg);
return NULL;
}
opal_memchecker_base_mem_defined (reg->rcache_context, bound - base);
mca_rcache_base_vma_insert (vma_module, reg, 0);
}
}
opal_atomic_wmb ();
*local_ptr = (void *) ((uintptr_t) reg->rcache_context +
(ptrdiff_t)((uintptr_t) rem_ptr - (uintptr_t) reg->base));
return reg;
}
/*
* User ioctl to the XPMEM driver. Only 64-bit user applications are
* supported.
*/
static long
xpmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
long ret;
switch (cmd) {
case XPMEM_CMD_VERSION: {
return XPMEM_CURRENT_VERSION;
}
case XPMEM_CMD_MAKE: {
struct xpmem_cmd_make make_info;
xpmem_segid_t segid;
if (copy_from_user(&make_info, (void __user *)arg,
sizeof(struct xpmem_cmd_make)))
return -EFAULT;
ret = xpmem_make(make_info.vaddr, make_info.size,
make_info.permit_type,
(void *)make_info.permit_value, &segid);
if (ret != 0)
return ret;
if (put_user(segid,
&((struct xpmem_cmd_make __user *)arg)->segid)) {
(void)xpmem_remove(segid);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_REMOVE: {
struct xpmem_cmd_remove remove_info;
if (copy_from_user(&remove_info, (void __user *)arg,
sizeof(struct xpmem_cmd_remove)))
return -EFAULT;
return xpmem_remove(remove_info.segid);
}
case XPMEM_CMD_GET: {
struct xpmem_cmd_get get_info;
xpmem_apid_t apid;
if (copy_from_user(&get_info, (void __user *)arg,
sizeof(struct xpmem_cmd_get)))
return -EFAULT;
ret = xpmem_get(get_info.segid, get_info.flags,
get_info.permit_type,
(void *)get_info.permit_value, &apid);
if (ret != 0)
return ret;
if (put_user(apid,
&((struct xpmem_cmd_get __user *)arg)->apid)) {
(void)xpmem_release(apid);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_RELEASE: {
struct xpmem_cmd_release release_info;
if (copy_from_user(&release_info, (void __user *)arg,
sizeof(struct xpmem_cmd_release)))
return -EFAULT;
return xpmem_release(release_info.apid);
}
case XPMEM_CMD_ATTACH: {
struct xpmem_cmd_attach attach_info;
u64 at_vaddr;
if (copy_from_user(&attach_info, (void __user *)arg,
sizeof(struct xpmem_cmd_attach)))
return -EFAULT;
ret = xpmem_attach(file, attach_info.apid, attach_info.offset,
attach_info.size, attach_info.vaddr,
attach_info.fd, attach_info.flags,
&at_vaddr);
if (ret != 0)
return ret;
if (put_user(at_vaddr,
&((struct xpmem_cmd_attach __user *)arg)->vaddr)) {
(void)xpmem_detach(at_vaddr);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_DETACH: {
struct xpmem_cmd_detach detach_info;
if (copy_from_user(&detach_info, (void __user *)arg,
sizeof(struct xpmem_cmd_detach)))
return -EFAULT;
return xpmem_detach(detach_info.vaddr);
}
case XPMEM_CMD_FORK_BEGIN: {
return xpmem_fork_begin();
}
case XPMEM_CMD_FORK_END: {
return xpmem_fork_end();
}
default:
break;
}
return -ENOIOCTLCMD;
}