/* Given a proc and a user virtual address, gives us the KVA. Useful for * debugging. Returns 0 if the page is unmapped (page lookup fails). This * doesn't play nice with Jumbo pages. */ uintptr_t uva2kva(struct proc *p, void *uva, size_t len, int prot) { struct page *u_page; uintptr_t offset = PGOFF(uva); if (!p) return 0; if (prot & PROT_WRITE) { if (!is_user_rwaddr(uva, len)) return 0; } else { if (!is_user_raddr(uva, len)) return 0; } u_page = page_lookup(p->env_pgdir, uva, 0); if (!u_page) return 0; return (uintptr_t)page2kva(u_page) + offset; }
/* Posts a message to the mbox, subject to flags. Feel free to send 0 for the * flags if you don't want to give them the option of EVENT_NOMSG (which is what * we do when sending an indirection event). Make sure that if mbox is a user * pointer, that you've checked it *and* have that processes address space * loaded. This can get called with a KVA for mbox. */ static void post_ev_msg(struct event_mbox *mbox, struct event_msg *msg, int ev_flags) { printd("Sending event type %d\n", msg->ev_type); /* Sanity check */ if (is_user_rwaddr(mbox)) assert(current); /* If they just want a bit (NOMSG), just set the bit */ if (ev_flags & EVENT_NOMSG) { SET_BITMASK_BIT_ATOMIC(mbox->ev_bitmap, msg->ev_type); } else { /* Enqueue returns 0 on success. On failure, set a bit. */ if (bcq_enqueue(&mbox->ev_msgs, msg, NR_BCQ_EVENTS, NR_BCQ_EV_LOOPS)) { atomic_inc((atomic_t)&mbox->ev_overflows); // careful here SET_BITMASK_BIT_ATOMIC(mbox->ev_bitmap, msg->ev_type); /* Catch "lots" of overflows that aren't acknowledged */ if (mbox->ev_overflows > 10000) warn("proc %d has way too many overflows", current->pid); } } }
/* Send an event to ev_q, based on the parameters in ev_q's flag. We don't * accept null ev_qs, since the caller ought to be checking before bothering to * make a msg and send it to the event_q. Vcoreid is who the kernel thinks the * message ought to go to (for IPIs). Appropriate for things like * EV_PREEMPT_PENDING, where we tell the affected vcore. To have the message go * where the kernel suggests, set EVENT_VCORE_APPRO(priate). */ void send_event(struct proc *p, struct event_queue *ev_q, struct event_msg *msg, uint32_t vcoreid) { struct per_cpu_info *pcpui = &per_cpu_info[core_id()]; struct proc *old_proc = pcpui->cur_proc; /* uncounted ref */ struct event_mbox *ev_mbox = 0, *vcore_mbox; struct event_msg local_msg = {0}; assert(p); if (!ev_q) { warn("[kernel] Null ev_q - kernel code should check before sending!"); return; } if (!is_user_rwaddr(ev_q)) { /* Ought to kill them, just warn for now */ warn("[kernel] Illegal addr for ev_q"); return; } /* ev_q can be a user pointer (not in procdata), so we need to make sure * we're in the right address space */ if (old_proc != p) { /* Technically, we're storing a ref here, but our current ref on p is * sufficient (so long as we don't decref below) */ pcpui->cur_proc = p; lcr3(p->env_cr3); } /* Get the mbox and vcoreid */ /* If we're going with APPRO, we use the kernel's suggested vcore's ev_mbox. * vcoreid is already what the kernel suggests. */ if (ev_q->ev_flags & EVENT_VCORE_APPRO) { ev_mbox = get_proc_ev_mbox(p, vcoreid); } else { /* common case */ ev_mbox = ev_q->ev_mbox; vcoreid = ev_q->ev_vcore; } /* Check on the style, which could affect our mbox selection. Other styles * would go here (or in similar functions we call to). Important thing is * we come out knowing which vcore to send to in the event of an IPI, and we * know what mbox to post to. */ if (ev_q->ev_flags & EVENT_ROUNDROBIN) { /* Pick a vcore, and if we don't have a mbox yet, pick that vcore's * default mbox. Assuming ev_vcore was the previous one used. Note * that round-robin overrides the passed-in vcoreid. */ vcoreid = (ev_q->ev_vcore + 1) % p->procinfo->num_vcores; ev_q->ev_vcore = vcoreid; if (!ev_mbox) ev_mbox = get_proc_ev_mbox(p, vcoreid); } /* At this point, we ought to have the right mbox to send the msg to, and * which vcore to send an IPI to (if we send one). The mbox could be the * vcore's vcpd ev_mbox. */ if (!ev_mbox) { /* this is a process error */ warn("[kernel] ought to have an mbox by now!"); goto out; } vcore_mbox = get_proc_ev_mbox(p, vcoreid); /* Allows the mbox to be the right vcoreid mbox (a KVA in procdata), or any * other user RW location. */ if ((ev_mbox != vcore_mbox) && (!is_user_rwaddr(ev_mbox))) { /* Ought to kill them, just warn for now */ warn("[kernel] Illegal addr for ev_mbox"); goto out; } /* We used to support no msgs, but quit being lazy and send a msg */ assert(msg); post_ev_msg(ev_mbox, msg, ev_q->ev_flags); /* Optional IPIs */ if (ev_q->ev_flags & EVENT_IPI) { /* if the mbox we sent to isn't the default one, we need to send the * vcore an ev_q indirection event */ if ((ev_mbox != vcore_mbox) && (!uva_is_kva(p, ev_mbox, vcore_mbox))) { /* it is tempting to send_kernel_event(), using the ev_q for that * event, but that is inappropriate here, since we are sending to a * specific vcore */ local_msg.ev_type = EV_EVENT; local_msg.ev_arg3 = ev_q; post_ev_msg(vcore_mbox, &local_msg, 0); } proc_notify(p, vcoreid); } out: /* Return to the old address space. We switched to p in the first place if * it wasn't the same as the original current (old_proc). */ if (old_proc != p) { pcpui->cur_proc = old_proc; if (old_proc) lcr3(old_proc->env_cr3); else lcr3(boot_cr3); } }
/* Send an event to ev_q, based on the parameters in ev_q's flag. We don't * accept null ev_qs, since the caller ought to be checking before bothering to * make a msg and send it to the event_q. Vcoreid is who the kernel thinks the * message ought to go to (for IPIs). Appropriate for things like * EV_PREEMPT_PENDING, where we tell the affected vcore. To have the message go * where the kernel suggests, set EVENT_VCORE_APPRO(priate). */ void send_event(struct proc *p, struct event_queue *ev_q, struct event_msg *msg, uint32_t vcoreid) { struct proc *old_proc; struct event_mbox *ev_mbox = 0; assert(p); printd("[kernel] sending msg to proc %p, ev_q %p\n", p, ev_q); if (!ev_q) { warn("[kernel] Null ev_q - kernel code should check before sending!"); return; } if (!is_user_rwaddr(ev_q, sizeof(struct event_queue))) { /* Ought to kill them, just warn for now */ printk("[kernel] Illegal addr for ev_q\n"); return; } /* This should be caught by "future technology" that can tell when the * kernel PFs on the user's behalf. For now, we catch common userspace bugs * (had this happen a few times). */ if (!PTE_ADDR(ev_q)) { printk("[kernel] Bad addr %p for ev_q\n", ev_q); return; } /* ev_q is a user pointer, so we need to make sure we're in the right * address space */ old_proc = switch_to(p); /* If we're an _S, just spam vcore0, and wake up if necessary. */ if (!__proc_is_mcp(p)) { spam_vcore(p, 0, msg, ev_q->ev_flags); wrmb(); /* don't let the notif_pending write pass the state read */ /* using the same pattern as in spam_public (which can have multiple * unblock callbacks */ if (p->state == PROC_WAITING) proc_wakeup(p); goto out; } /* Get the vcoreid that we'll message (if appropriate). For INDIR and * SPAMMING, this is the first choice of a vcore, but other vcores might get * it. Common case is !APPRO and !ROUNDROBIN. Note we are clobbering the * vcoreid parameter. */ if (!(ev_q->ev_flags & EVENT_VCORE_APPRO)) vcoreid = ev_q->ev_vcore; /* use the ev_q's vcoreid */ /* Note that RR overwrites APPRO */ if (ev_q->ev_flags & EVENT_ROUNDROBIN) { /* Pick a vcore, round-robin style. Assuming ev_vcore was the previous * one used. Note that round-robin overrides the passed-in vcoreid. * Also note this may be 'wrong' if num_vcores changes. */ vcoreid = (ev_q->ev_vcore + 1) % p->procinfo->num_vcores; ev_q->ev_vcore = vcoreid; } if (!vcoreid_is_safe(vcoreid)) { /* Ought to kill them, just warn for now */ printk("[kernel] Vcoreid %d unsafe! (too big?)\n", vcoreid); goto out; } /* If we're a SPAM_PUBLIC, they just want us to spam the message. Note we * don't care about the mbox, since it'll go to VCPD public mboxes, and * we'll prefer to send it to whatever vcoreid we determined at this point * (via APPRO or whatever). */ if (ev_q->ev_flags & EVENT_SPAM_PUBLIC) { spam_public_msg(p, msg, vcoreid, ev_q->ev_flags & EVENT_SPAM_FLAGS); goto out; } /* We aren't spamming and we know the default vcore, and now we need to * figure out which mbox to use. If they provided an mbox, we'll use it. * If not, we'll use a VCPD mbox (public or private, depending on the * flags). */ ev_mbox = ev_q->ev_mbox; if (!ev_mbox) ev_mbox = get_vcpd_mbox(vcoreid, ev_q->ev_flags); /* At this point, we ought to have the right mbox to send the msg to, and * which vcore to alert (IPI/INDIR) (if applicable). The mbox could be the * vcore's vcpd ev_mbox. */ if (!ev_mbox) { /* This shouldn't happen any more, this is more for sanity's sake */ warn("[kernel] ought to have an mbox by now!"); goto out; } /* Even if we're using an mbox in procdata (VCPD), we want a user pointer */ if (!is_user_rwaddr(ev_mbox, sizeof(struct event_mbox))) { /* Ought to kill them, just warn for now */ printk("[kernel] Illegal addr for ev_mbox\n"); goto out; } /* We used to support no msgs, but quit being lazy and send a 'msg'. If the * ev_q is a NOMSG, we won't actually memcpy or anything, it'll just be a * vehicle for sending the ev_type. */ assert(msg); post_ev_msg(p, ev_mbox, msg, ev_q->ev_flags); wmb(); /* ensure ev_msg write is before alerting the vcore */ /* Prod/alert a vcore with an IPI or INDIR, if desired. INDIR will also * call try_notify (IPI) later */ if (ev_q->ev_flags & EVENT_INDIR) { send_indir(p, ev_q, vcoreid); } else { /* they may want an IPI despite not wanting an INDIR */ try_notify(p, vcoreid, ev_q->ev_flags); } /* Fall through */ out: /* Return to the old address space. */ switch_back(p, old_proc); }