/* Writes the msg to the vcpd/default mbox of the vcore. Doesn't need to check
* for current, or care about what the process wants. */
void post_vcore_event(struct proc *p, struct event_msg *msg, uint32_t vcoreid)
{
struct event_mbox *vcore_mbox;
/* kernel address of the vcpd mbox */
vcore_mbox = get_proc_ev_mbox(p, vcoreid);
post_ev_msg(vcore_mbox, msg, 0); /* no chance for a NOMSG either */
}
/* Helper: use this when sending a message to a VCPD mbox. It just posts to the
* ev_mbox and sets notif pending. Note this uses a userspace address for the
* VCPD (though not a user's pointer). */
static void post_vc_msg(struct proc *p, uint32_t vcoreid,
struct event_mbox *ev_mbox, struct event_msg *ev_msg,
int ev_flags)
{
struct preempt_data *vcpd = &__procdata.vcore_preempt_data[vcoreid];
post_ev_msg(p, ev_mbox, ev_msg, ev_flags);
/* Set notif pending so userspace doesn't miss the message while yielding */
wmb(); /* Ensure ev_msg write is before notif_pending */
/* proc_notify() also sets this, but the ev_q might not have requested an
* IPI, so we have to do it here too. */
vcpd->notif_pending = TRUE;
}
/* Send an event to ev_q, based on the parameters in ev_q's flag. We don't
* accept null ev_qs, since the caller ought to be checking before bothering to
* make a msg and send it to the event_q. Vcoreid is who the kernel thinks the
* message ought to go to (for IPIs). Appropriate for things like
* EV_PREEMPT_PENDING, where we tell the affected vcore. To have the message go
* where the kernel suggests, set EVENT_VCORE_APPRO(priate). */
void send_event(struct proc *p, struct event_queue *ev_q, struct event_msg *msg,
uint32_t vcoreid)
{
struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
struct proc *old_proc = pcpui->cur_proc; /* uncounted ref */
struct event_mbox *ev_mbox = 0, *vcore_mbox;
struct event_msg local_msg = {0};
assert(p);
if (!ev_q) {
warn("[kernel] Null ev_q - kernel code should check before sending!");
return;
}
if (!is_user_rwaddr(ev_q)) {
/* Ought to kill them, just warn for now */
warn("[kernel] Illegal addr for ev_q");
return;
}
/* ev_q can be a user pointer (not in procdata), so we need to make sure
* we're in the right address space */
if (old_proc != p) {
/* Technically, we're storing a ref here, but our current ref on p is
* sufficient (so long as we don't decref below) */
pcpui->cur_proc = p;
lcr3(p->env_cr3);
}
/* Get the mbox and vcoreid */
/* If we're going with APPRO, we use the kernel's suggested vcore's ev_mbox.
* vcoreid is already what the kernel suggests. */
if (ev_q->ev_flags & EVENT_VCORE_APPRO) {
ev_mbox = get_proc_ev_mbox(p, vcoreid);
} else { /* common case */
ev_mbox = ev_q->ev_mbox;
vcoreid = ev_q->ev_vcore;
}
/* Check on the style, which could affect our mbox selection. Other styles
* would go here (or in similar functions we call to). Important thing is
* we come out knowing which vcore to send to in the event of an IPI, and we
* know what mbox to post to. */
if (ev_q->ev_flags & EVENT_ROUNDROBIN) {
/* Pick a vcore, and if we don't have a mbox yet, pick that vcore's
* default mbox. Assuming ev_vcore was the previous one used. Note
* that round-robin overrides the passed-in vcoreid. */
vcoreid = (ev_q->ev_vcore + 1) % p->procinfo->num_vcores;
ev_q->ev_vcore = vcoreid;
if (!ev_mbox)
ev_mbox = get_proc_ev_mbox(p, vcoreid);
}
/* At this point, we ought to have the right mbox to send the msg to, and
* which vcore to send an IPI to (if we send one). The mbox could be the
* vcore's vcpd ev_mbox. */
if (!ev_mbox) {
/* this is a process error */
warn("[kernel] ought to have an mbox by now!");
goto out;
}
vcore_mbox = get_proc_ev_mbox(p, vcoreid);
/* Allows the mbox to be the right vcoreid mbox (a KVA in procdata), or any
* other user RW location. */
if ((ev_mbox != vcore_mbox) && (!is_user_rwaddr(ev_mbox))) {
/* Ought to kill them, just warn for now */
warn("[kernel] Illegal addr for ev_mbox");
goto out;
}
/* We used to support no msgs, but quit being lazy and send a msg */
assert(msg);
post_ev_msg(ev_mbox, msg, ev_q->ev_flags);
/* Optional IPIs */
if (ev_q->ev_flags & EVENT_IPI) {
/* if the mbox we sent to isn't the default one, we need to send the
* vcore an ev_q indirection event */
if ((ev_mbox != vcore_mbox) && (!uva_is_kva(p, ev_mbox, vcore_mbox))) {
/* it is tempting to send_kernel_event(), using the ev_q for that
* event, but that is inappropriate here, since we are sending to a
* specific vcore */
local_msg.ev_type = EV_EVENT;
local_msg.ev_arg3 = ev_q;
post_ev_msg(vcore_mbox, &local_msg, 0);
}
proc_notify(p, vcoreid);
}
out:
/* Return to the old address space. We switched to p in the first place if
* it wasn't the same as the original current (old_proc). */
if (old_proc != p) {
pcpui->cur_proc = old_proc;
if (old_proc)
lcr3(old_proc->env_cr3);
else
lcr3(boot_cr3);
}
}
/* Send an event to ev_q, based on the parameters in ev_q's flag. We don't
* accept null ev_qs, since the caller ought to be checking before bothering to
* make a msg and send it to the event_q. Vcoreid is who the kernel thinks the
* message ought to go to (for IPIs). Appropriate for things like
* EV_PREEMPT_PENDING, where we tell the affected vcore. To have the message go
* where the kernel suggests, set EVENT_VCORE_APPRO(priate). */
void send_event(struct proc *p, struct event_queue *ev_q, struct event_msg *msg,
uint32_t vcoreid)
{
struct proc *old_proc;
struct event_mbox *ev_mbox = 0;
assert(p);
printd("[kernel] sending msg to proc %p, ev_q %p\n", p, ev_q);
if (!ev_q) {
warn("[kernel] Null ev_q - kernel code should check before sending!");
return;
}
if (!is_user_rwaddr(ev_q, sizeof(struct event_queue))) {
/* Ought to kill them, just warn for now */
printk("[kernel] Illegal addr for ev_q\n");
return;
}
/* This should be caught by "future technology" that can tell when the
* kernel PFs on the user's behalf. For now, we catch common userspace bugs
* (had this happen a few times). */
if (!PTE_ADDR(ev_q)) {
printk("[kernel] Bad addr %p for ev_q\n", ev_q);
return;
}
/* ev_q is a user pointer, so we need to make sure we're in the right
* address space */
old_proc = switch_to(p);
/* If we're an _S, just spam vcore0, and wake up if necessary. */
if (!__proc_is_mcp(p)) {
spam_vcore(p, 0, msg, ev_q->ev_flags);
wrmb(); /* don't let the notif_pending write pass the state read */
/* using the same pattern as in spam_public (which can have multiple
* unblock callbacks */
if (p->state == PROC_WAITING)
proc_wakeup(p);
goto out;
}
/* Get the vcoreid that we'll message (if appropriate). For INDIR and
* SPAMMING, this is the first choice of a vcore, but other vcores might get
* it. Common case is !APPRO and !ROUNDROBIN. Note we are clobbering the
* vcoreid parameter. */
if (!(ev_q->ev_flags & EVENT_VCORE_APPRO))
vcoreid = ev_q->ev_vcore; /* use the ev_q's vcoreid */
/* Note that RR overwrites APPRO */
if (ev_q->ev_flags & EVENT_ROUNDROBIN) {
/* Pick a vcore, round-robin style. Assuming ev_vcore was the previous
* one used. Note that round-robin overrides the passed-in vcoreid.
* Also note this may be 'wrong' if num_vcores changes. */
vcoreid = (ev_q->ev_vcore + 1) % p->procinfo->num_vcores;
ev_q->ev_vcore = vcoreid;
}
if (!vcoreid_is_safe(vcoreid)) {
/* Ought to kill them, just warn for now */
printk("[kernel] Vcoreid %d unsafe! (too big?)\n", vcoreid);
goto out;
}
/* If we're a SPAM_PUBLIC, they just want us to spam the message. Note we
* don't care about the mbox, since it'll go to VCPD public mboxes, and
* we'll prefer to send it to whatever vcoreid we determined at this point
* (via APPRO or whatever). */
if (ev_q->ev_flags & EVENT_SPAM_PUBLIC) {
spam_public_msg(p, msg, vcoreid, ev_q->ev_flags & EVENT_SPAM_FLAGS);
goto out;
}
/* We aren't spamming and we know the default vcore, and now we need to
* figure out which mbox to use. If they provided an mbox, we'll use it.
* If not, we'll use a VCPD mbox (public or private, depending on the
* flags). */
ev_mbox = ev_q->ev_mbox;
if (!ev_mbox)
ev_mbox = get_vcpd_mbox(vcoreid, ev_q->ev_flags);
/* At this point, we ought to have the right mbox to send the msg to, and
* which vcore to alert (IPI/INDIR) (if applicable). The mbox could be the
* vcore's vcpd ev_mbox. */
if (!ev_mbox) {
/* This shouldn't happen any more, this is more for sanity's sake */
warn("[kernel] ought to have an mbox by now!");
goto out;
}
/* Even if we're using an mbox in procdata (VCPD), we want a user pointer */
if (!is_user_rwaddr(ev_mbox, sizeof(struct event_mbox))) {
/* Ought to kill them, just warn for now */
printk("[kernel] Illegal addr for ev_mbox\n");
goto out;
}
/* We used to support no msgs, but quit being lazy and send a 'msg'. If the
* ev_q is a NOMSG, we won't actually memcpy or anything, it'll just be a
* vehicle for sending the ev_type. */
assert(msg);
post_ev_msg(p, ev_mbox, msg, ev_q->ev_flags);
wmb(); /* ensure ev_msg write is before alerting the vcore */
/* Prod/alert a vcore with an IPI or INDIR, if desired. INDIR will also
* call try_notify (IPI) later */
if (ev_q->ev_flags & EVENT_INDIR) {
send_indir(p, ev_q, vcoreid);
} else {
/* they may want an IPI despite not wanting an INDIR */
try_notify(p, vcoreid, ev_q->ev_flags);
}
/* Fall through */
out:
/* Return to the old address space. */
switch_back(p, old_proc);
}
