static int
xpmem_try_get_remote(struct xpmem_thread_group * seg_tg,
xpmem_segid_t segid,
int flags,
int permit_type,
void * permit_value)
{
xpmem_apid_t apid = 0;
xpmem_domid_t domid = 0;
xpmem_sigid_t sigid = 0;
size_t size = 0;
int status = 0;
int seg_flags = 0;
status = xpmem_get_remote(
xpmem_my_part->domain_link,
segid,
flags,
permit_type,
(u64)permit_value,
&apid,
(u64 *)&size,
&domid,
&sigid);
if (status != 0)
return status;
if (apid == -1)
return -1;
if (size > 0)
seg_flags |= XPMEM_MEM_MODE;
if (sigid != 0)
seg_flags |= XPMEM_SIG_MODE;
if (seg_flags == 0) {
XPMEM_ERR("Creating shadow segment that is neither a memory segment nor signalable! This should be impossible");
xpmem_release_remote(xpmem_my_part->domain_link, segid, apid);
return -EINVAL;
}
seg_flags |= XPMEM_FLAG_SHADOW;
/* We've been given a remote apid. The strategy is to fake like the segid
* was created locally by creating a "shadow" segment ourselves
*/
status = xpmem_make_segment(0, size, permit_type, permit_value,
seg_flags, seg_tg, segid, apid, domid, sigid, NULL);
if (status != 0) {
XPMEM_ERR("Unable to create shadow segment");
xpmem_release_remote(xpmem_my_part->domain_link, segid, apid);
return status;
}
/* Success */
return 0;
}
static void
__xpmem_detach_att(struct xpmem_access_permit * ap,
struct xpmem_attachment * att)
{
aspace_mapping_t mapping;
int status, index;
/* No address space to update on remote attachments - they are purely for bookkeeping */
if (!(att->flags & XPMEM_FLAG_REMOTE)) {
/* Lookup aspace mapping */
status = aspace_lookup_mapping(ap->tg->aspace->id, att->at_vaddr, &mapping);
if (status != 0) {
XPMEM_ERR("aspace_lookup_mapping() failed (%d)", status);
return;
}
/* On shadow attachments, we added a new aspace region. Remove it now */
if (ap->seg->flags & XPMEM_FLAG_SHADOW) {
BUG_ON(mapping.flags & VM_SMARTMAP);
/* Remove aspace mapping */
status = aspace_del_region(ap->tg->aspace->id, mapping.start, mapping.end - mapping.start);
if (status != 0) {
XPMEM_ERR("aspace_del_region() failed (%d)", status);
return;
}
/* Perform remote detachment */
xpmem_detach_remote(xpmem_my_part->domain_link, ap->seg->segid, ap->seg->remote_apid, att->at_vaddr);
}
else {
/* If this was a real attachment, it should be using SMARTMAP */
BUG_ON(!(mapping.flags & VM_SMARTMAP));
}
/* Remove from att hash list - only done on local memory */
index = xpmem_att_hashtable_index(att->at_vaddr);
write_lock(&ap->tg->att_hashtable[index].lock);
list_del(&att->att_hashnode);
write_unlock(&ap->tg->att_hashtable[index].lock);
}
/* Remove from ap list */
spin_lock(&ap->lock);
list_del_init(&att->att_node);
spin_unlock(&ap->lock);
}
/* Pong all of the connections we have notifying path to the nameserver,
* skipping id 'skip'
*/
static void
xpmem_pong_ns(struct xpmem_partition_state * part_state,
xpmem_link_t skip)
{
struct xpmem_cmd_ex pong_cmd;
memset(&(pong_cmd), 0, sizeof(struct xpmem_cmd_ex));
pong_cmd.type = XPMEM_PONG_NS;
pong_cmd.req_dom = -1;
pong_cmd.src_dom = -1;
pong_cmd.dst_dom = -1;
{
int i = 0;
for (i = 0; i <= XPMEM_MAX_LINK_ID; i++) {
xpmem_link_t search_id = (xpmem_link_t)i;
if (search_id == skip) {
continue;
}
/* Don't PONG the local domain */
if (search_id == part_state->local_link) {
continue;
}
if (xpmem_search_link(part_state, search_id)) {
if (xpmem_send_cmd_link(part_state, search_id, &pong_cmd)) {
XPMEM_ERR("Cannot send PONG on link %lli", search_id);
}
}
}
}
}
/*
* Attach a remote XPMEM address segment
*/
static int
xpmem_try_attach_remote(xpmem_segid_t segid,
xpmem_apid_t apid,
off_t offset,
size_t size,
vaddr_t * vaddr)
{
int status = 0;
vaddr_t at_vaddr = 0;
/* Find free address space */
status = aspace_find_hole(current->aspace->id, 0, size, PAGE_SIZE, &at_vaddr);
if (status != 0) {
XPMEM_ERR("aspace_find_hole() failed (%d)", status);
return status;
}
/* Add region to aspace */
status = aspace_add_region(current->aspace->id, at_vaddr, size, VM_READ | VM_WRITE | VM_USER,
PAGE_SIZE, "xpmem");
if (status != 0) {
XPMEM_ERR("aspace_add_region() failed (%d)", status);
return status;
}
/* Attach to remote memory */
status = xpmem_attach_remote(xpmem_my_part->domain_link, segid, apid, offset, size, (u64)at_vaddr);
if (status != 0) {
XPMEM_ERR("xpmem_attach_remote() failed (%d)", status);
aspace_del_region(current->aspace->id, at_vaddr, size);
return status;
}
*vaddr = at_vaddr;
return 0;
}
/* Process a regular XPMEM command. If we get here we are connected to the name
* server already and have a domid
*/
static int
xpmem_fwd_process_xpmem_cmd(struct xpmem_partition_state * part_state,
xpmem_link_t link,
struct xpmem_cmd_ex * cmd)
{
/* There's no reason not to reuse the input command struct for responses */
struct xpmem_cmd_ex * out_cmd = cmd;
xpmem_link_t out_link = link;
/* If we don't have a domid, we have to fail */
if (part_state->domid <= 0) {
XPMEM_ERR("This domain has no XPMEM domid. Are you running the nameserver anywhere?");
xpmem_set_failure(out_cmd);
xpmem_set_complete(out_cmd);
if (xpmem_send_cmd_link(part_state, out_link, out_cmd)) {
XPMEM_ERR("Cannot send command on link %lli", out_link);
}
return -EFAULT;
}
switch (cmd->type) {
case XPMEM_MAKE:
case XPMEM_SEARCH:
case XPMEM_REMOVE:
case XPMEM_GET:
case XPMEM_RELEASE:
case XPMEM_ATTACH:
case XPMEM_DETACH:
case XPMEM_MAKE_COMPLETE:
case XPMEM_SEARCH_COMPLETE:
case XPMEM_REMOVE_COMPLETE:
case XPMEM_GET_COMPLETE:
case XPMEM_RELEASE_COMPLETE:
case XPMEM_ATTACH_COMPLETE:
case XPMEM_DETACH_COMPLETE: {
out_link = xpmem_search_domid(part_state, out_cmd->dst_dom);
if (out_link == 0) {
XPMEM_ERR("Cannot find domid %lli in hashtable", out_cmd->dst_dom);
return -EINVAL;
}
break;
}
default: {
XPMEM_ERR("Unknown operation: %s", cmd_to_string(cmd->type));
return -EINVAL;
}
}
/* Write the response */
if (xpmem_send_cmd_link(part_state, out_link, out_cmd)) {
XPMEM_ERR("Cannot send command on link %lli", out_link);
return -EFAULT;
}
return 0;
}
/* Process an XPMEM_DOMID_REQUEST/RESPONSE/RELEASE command */
static int
xpmem_fwd_process_domid_cmd(struct xpmem_partition_state * part_state,
xpmem_link_t link,
struct xpmem_cmd_ex * cmd)
{
struct xpmem_fwd_state * fwd_state = part_state->fwd_state;
/* There's no reason not to reuse the input command struct for responses */
struct xpmem_cmd_ex * out_cmd = cmd;
xpmem_link_t out_link = link;
switch (cmd->type) {
case XPMEM_DOMID_REQUEST: {
/* A domid is requested by someone downstream from us on link
* 'link'. If we can't reach the nameserver, just return failure,
* because the request should not come through us unless we have a
* route already
*/
if (!xpmem_have_ns_link(fwd_state)) {
return -1;
}
/* Buffer the request */
{
struct xpmem_domid_req_iter * iter = NULL;
unsigned long flags = 0;
iter = kmem_alloc(sizeof(struct xpmem_domid_req_iter));
if (!iter) {
return -ENOMEM;
}
iter->link = link;
spin_lock_irqsave(&(fwd_state->lock), flags);
{
list_add_tail(&(iter->node), &(fwd_state->domid_req_list));
}
spin_unlock_irqrestore(&(fwd_state->lock), flags);
/* Forward request up to the nameserver */
out_link = fwd_state->ns_link;
}
break;
}
case XPMEM_DOMID_RESPONSE: {
int ret = 0;
/* We've been allocated a domid.
*
* If our domain has no domid, take it for ourselves it.
* Otherwise, assign it to a link that has requested a domid from us
*/
if (part_state->domid <= 0) {
part_state->domid = cmd->domid_req.domid;
/* Update the domid map to remember our own domid */
ret = xpmem_add_domid(part_state, part_state->domid, part_state->local_link);
if (ret == 0) {
XPMEM_ERR("Cannot insert domid %lli into hashtable", part_state->domid);
return -EFAULT;
}
return 0;
} else {
struct xpmem_domid_req_iter * iter = NULL;
unsigned long flags = 0;
if (list_empty(&(fwd_state->domid_req_list))) {
XPMEM_ERR("We currently do not support the buffering of XPMEM domids");
return -1;
}
spin_lock_irqsave(&(fwd_state->lock), flags);
{
iter = list_first_entry(&(fwd_state->domid_req_list),
struct xpmem_domid_req_iter,
node);
list_del(&(iter->node));
}
spin_unlock_irqrestore(&(fwd_state->lock), flags);
/* Forward the domid to this link */
out_link = iter->link;
kmem_free(iter);
/* Update the domid map to remember who has this */
ret = xpmem_add_domid(part_state, cmd->domid_req.domid, out_link);
if (ret == 0) {
XPMEM_ERR("Cannot insert domid %lli into hashtable", cmd->domid_req.domid);
return -EFAULT;
}
}
break;
}
case XPMEM_DOMID_RELEASE:
/* Someone downstream is releasing their domid: simply forward to the
* namserver */
out_link = xpmem_search_domid(part_state, out_cmd->dst_dom);
if (out_link == 0) {
XPMEM_ERR("Cannot find domid %lli in hashtable", out_cmd->dst_dom);
return -EFAULT;
}
break;
default: {
XPMEM_ERR("Unknown DOMID operation: %s", cmd_to_string(cmd->type));
return -EINVAL;
}
}
/* Send the response */
if (xpmem_send_cmd_link(part_state, out_link, out_cmd)) {
XPMEM_ERR("Cannot send command on link %lli", out_link);
return -EFAULT;
}
return 0;
}
/* Process an XPMEM_PING/PONG_NS command */
static int
xpmem_fwd_process_ping_cmd(struct xpmem_partition_state * part_state,
xpmem_link_t link,
struct xpmem_cmd_ex * cmd)
{
struct xpmem_fwd_state * fwd_state = part_state->fwd_state;
switch (cmd->type) {
case XPMEM_PING_NS: {
/* Do we know the way to the nameserver that is not through the link
* pinging us? If we do, respond with a PONG.
*/
if (xpmem_have_ns_link(fwd_state)) {
/* Send PONG back to the source */
cmd->type = XPMEM_PONG_NS;
if (xpmem_send_cmd_link(part_state, link, cmd)) {
printk(KERN_ERR "XPMEM: cannot send command on link %lli", link);
return -EFAULT;
}
}
break;
}
case XPMEM_PONG_NS: {
unsigned long flags = 0;
int ret = 0;
int req = 0;
/* We received a PONG. So, the nameserver can be found through this
* link
*/
/* Remember the link */
spin_lock_irqsave(&(fwd_state->lock), flags);
{
fwd_state->ns_link = link;
req = fwd_state->domid_requested;
if (req == 0) {
fwd_state->domid_requested = 1;
}
}
spin_unlock_irqrestore(&(fwd_state->lock), flags);
/* Update the domid map to remember this link */
ret = xpmem_add_domid(part_state, XPMEM_NS_DOMID, link);
if (ret == 0) {
XPMEM_ERR("Cannot insert domid %lli into hashtable", (xpmem_domid_t)XPMEM_NS_DOMID);
return -EFAULT;
}
/* Broadcast the PONG to all our neighbors, except the source */
xpmem_pong_ns(part_state, link);
/* Have we requested a domid */
if (req == 0) {
struct xpmem_cmd_ex domid_req;
memset(&(domid_req), 0, sizeof(struct xpmem_cmd_ex));
domid_req.type = XPMEM_DOMID_REQUEST;
domid_req.req_dom = -1;
domid_req.src_dom = -1;
domid_req.dst_dom = XPMEM_NS_DOMID;
if (xpmem_send_cmd_link(part_state, fwd_state->ns_link, &domid_req)) {
XPMEM_ERR("Cannot send command on link %lli", fwd_state->ns_link);
return -EFAULT;
}
}
break;
}
default: {
XPMEM_ERR("Unknown PING operation: %s", cmd_to_string(cmd->type));
return -EINVAL;
}
}
return 0;
}
