int
shmem_transport_xpmem_fini(void)
{
int i, peer_num;
if (NULL != shmem_transport_xpmem_peers) {
for (i = 0 ; i < shmem_internal_num_pes; ++i) {
peer_num = SHMEM_GET_RANK_SAME_NODE(i);
if (-1 == peer_num) continue;
if (shmem_internal_my_pe == i) continue;
if (NULL != shmem_transport_xpmem_peers[peer_num].data_ptr) {
xpmem_detach(shmem_transport_xpmem_peers[peer_num].data_attach_ptr);
}
if (0 != shmem_transport_xpmem_peers[peer_num].data_apid) {
xpmem_release(shmem_transport_xpmem_peers[peer_num].data_apid);
}
if (NULL != shmem_transport_xpmem_peers[peer_num].heap_ptr) {
xpmem_detach(shmem_transport_xpmem_peers[peer_num].heap_attach_ptr);
}
if (0 != shmem_transport_xpmem_peers[peer_num].heap_apid) {
xpmem_release(shmem_transport_xpmem_peers[peer_num].heap_apid);
}
}
free(shmem_transport_xpmem_peers);
}
if (0 != my_info.data_seg) {
xpmem_remove(my_info.data_seg);
}
if (0 != my_info.heap_seg) {
xpmem_remove(my_info.heap_seg);
}
return 0;
}
static ucs_status_t uct_xpmem_dereg(uct_mm_id_t mmid)
{
int ret;
ret = xpmem_remove(mmid);
if (ret < 0) {
/* No error since there a chance that it already was released
* or deregistered */
ucs_debug("Failed to remove xpmem segment 0x%"PRIx64": %m", mmid);
}
ucs_trace("xpmem removed segment 0x%"PRIx64, mmid);
return UCS_OK;
}
/*
* Get permission to access a specified segid.
*/
int
xpmem_get(xpmem_segid_t segid, int flags, int permit_type, void *permit_value,
xpmem_apid_t *apid_p)
{
xpmem_apid_t apid = 0;
int status = 0;
int shadow_seg = 0;
struct xpmem_segment *seg = NULL;
struct xpmem_thread_group *ap_tg, *seg_tg;
if (segid <= 0)
return -EINVAL;
if ((flags & ~(XPMEM_RDONLY | XPMEM_RDWR)) ||
(flags & (XPMEM_RDONLY | XPMEM_RDWR)) ==
(XPMEM_RDONLY | XPMEM_RDWR))
return -EINVAL;
switch (permit_type) {
case XPMEM_PERMIT_MODE:
case XPMEM_GLOBAL_MODE:
if (permit_value != NULL)
return -EINVAL;
break;
default:
return -EINVAL;
}
/* There are 2 cases that result in a remote lookup:
* (1) The thread group encoded in the segment does not exist locally.
*
* (2) The thread group exists locally, but the segment does not. The
* ids for thread groups are not globally unique, so it's possible that
* the same thread group id exists in two separate enclaves, but only
* one will own the segment
*/
seg_tg = xpmem_tg_ref_by_segid(segid);
if (IS_ERR(seg_tg)) {
seg_tg = xpmem_tg_ref_by_gid(current->aspace->id);
if (IS_ERR(seg_tg))
return PTR_ERR(seg_tg);
shadow_seg = 1;
}
if (!shadow_seg) {
seg = xpmem_seg_ref_by_segid(seg_tg, segid);
if (IS_ERR(seg))
shadow_seg = 1;
}
if (shadow_seg) {
/* No local segment found. Look for a remote one */
/* NOTE: in either case, the tg has already been ref'd. We ref the
* current process' tg if no tg is found for the segid
*/
status = xpmem_try_get_remote(seg_tg, segid, flags, permit_type,
permit_value);
if (status != 0) {
xpmem_tg_deref(seg_tg);
return status;
}
/* Now, get the shadow segment */
seg = xpmem_seg_ref_by_segid(seg_tg, segid);
if (IS_ERR(seg)) {
/* Error should be impossible here, but we'll
* check anyway. The shadow segment was created in
* xpmem_try_get_remote, so destroy it here */
xpmem_remove(segid);
xpmem_tg_deref(seg_tg);
return PTR_ERR(seg);
}
}
/* find accessor's thread group structure */
ap_tg = xpmem_tg_ref_by_gid(current->aspace->id);
if (IS_ERR(ap_tg)) {
BUG_ON(PTR_ERR(ap_tg) != -ENOENT);
status = -XPMEM_ERRNO_NOPROC;
goto err_ap_tg;
}
apid = xpmem_make_apid(ap_tg);
if (apid < 0) {
status = apid;
goto err_apid;
}
status = xpmem_get_segment(flags, permit_type, permit_value, apid,
seg, seg_tg, ap_tg);
if (status != 0)
goto err_get;
*apid_p = apid;
xpmem_tg_deref(ap_tg);
/*
* The following two derefs
*
* xpmem_seg_deref(seg);
* xpmem_tg_deref(seg_tg);
*
* aren't being done at this time in order to prevent the seg
* and seg_tg structures from being prematurely kmem_free'd as long as the
* potential for them to be referenced via this ap structure exists.
*
* These two derefs will be done by xpmem_release_ap() at the time
* this ap structure is destroyed.
*/
return status;
err_get:
err_apid:
xpmem_tg_deref(ap_tg);
err_ap_tg:
/* If we created a shadow segment, destroy it on error. Else, just
* deref it
*/
if (shadow_seg)
xpmem_remove(segid);
else
xpmem_seg_deref(seg);
xpmem_tg_deref(seg_tg);
return status;
}
/*
* User ioctl to the XPMEM driver. Only 64-bit user applications are
* supported.
*/
static long
xpmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
long ret;
switch (cmd) {
case XPMEM_CMD_VERSION: {
return XPMEM_CURRENT_VERSION;
}
case XPMEM_CMD_MAKE: {
struct xpmem_cmd_make make_info;
xpmem_segid_t segid;
if (copy_from_user(&make_info, (void __user *)arg,
sizeof(struct xpmem_cmd_make)))
return -EFAULT;
ret = xpmem_make(make_info.vaddr, make_info.size,
make_info.permit_type,
(void *)make_info.permit_value, &segid);
if (ret != 0)
return ret;
if (put_user(segid,
&((struct xpmem_cmd_make __user *)arg)->segid)) {
(void)xpmem_remove(segid);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_REMOVE: {
struct xpmem_cmd_remove remove_info;
if (copy_from_user(&remove_info, (void __user *)arg,
sizeof(struct xpmem_cmd_remove)))
return -EFAULT;
return xpmem_remove(remove_info.segid);
}
case XPMEM_CMD_GET: {
struct xpmem_cmd_get get_info;
xpmem_apid_t apid;
if (copy_from_user(&get_info, (void __user *)arg,
sizeof(struct xpmem_cmd_get)))
return -EFAULT;
ret = xpmem_get(get_info.segid, get_info.flags,
get_info.permit_type,
(void *)get_info.permit_value, &apid);
if (ret != 0)
return ret;
if (put_user(apid,
&((struct xpmem_cmd_get __user *)arg)->apid)) {
(void)xpmem_release(apid);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_RELEASE: {
struct xpmem_cmd_release release_info;
if (copy_from_user(&release_info, (void __user *)arg,
sizeof(struct xpmem_cmd_release)))
return -EFAULT;
return xpmem_release(release_info.apid);
}
case XPMEM_CMD_ATTACH: {
struct xpmem_cmd_attach attach_info;
u64 at_vaddr;
if (copy_from_user(&attach_info, (void __user *)arg,
sizeof(struct xpmem_cmd_attach)))
return -EFAULT;
ret = xpmem_attach(file, attach_info.apid, attach_info.offset,
attach_info.size, attach_info.vaddr,
attach_info.fd, attach_info.flags,
&at_vaddr);
if (ret != 0)
return ret;
if (put_user(at_vaddr,
&((struct xpmem_cmd_attach __user *)arg)->vaddr)) {
(void)xpmem_detach(at_vaddr);
return -EFAULT;
}
return 0;
}
case XPMEM_CMD_DETACH: {
struct xpmem_cmd_detach detach_info;
if (copy_from_user(&detach_info, (void __user *)arg,
sizeof(struct xpmem_cmd_detach)))
return -EFAULT;
return xpmem_detach(detach_info.vaddr);
}
case XPMEM_CMD_FORK_BEGIN: {
return xpmem_fork_begin();
}
case XPMEM_CMD_FORK_END: {
return xpmem_fork_end();
}
default:
break;
}
return -ENOIOCTLCMD;
}