/// Destroy the local state associated with a given channel
void lmp_chan_destroy(struct lmp_chan *lc)
{
lc->connstate = LMP_DISCONNECTED;
cap_destroy(lc->local_cap);
if (lc->endpoint != NULL) {
lmp_endpoint_free(lc->endpoint);
}
// remove from send retry queue on dispatcher
if (waitset_chan_is_registered(&lc->send_waitset)) {
assert(lc->prev != NULL && lc->next != NULL);
dispatcher_handle_t handle = disp_disable();
struct dispatcher_generic *disp = get_dispatcher_generic(handle);
if (lc->next == lc->prev) {
assert_disabled(lc->next == lc);
assert_disabled(disp->lmp_send_events_list == lc);
disp->lmp_send_events_list = NULL;
} else {
lc->prev->next = lc->next;
lc->next->prev = lc->prev;
}
disp_enable(handle);
#ifndef NDEBUG
lc->next = lc->prev = NULL;
#endif
}
waitset_chanstate_destroy(&lc->send_waitset);
}
/**
* \brief Wakeup a thread on a foreign dispatcher while disabled.
*
* \param core_id Core ID to wakeup on
* \param thread Pointer to thread to wakeup
* \param mydisp Dispatcher this function is running on
*
* \return SYS_ERR_OK on success.
*/
static errval_t domain_wakeup_on_coreid_disabled(coreid_t core_id,
struct thread *thread,
dispatcher_handle_t mydisp)
{
struct domain_state *ds = get_domain_state();
// XXX: Ugly hack to allow waking up on a core id we don't have a
// dispatcher handler for
thread->coreid = core_id;
// Catch this early
assert_disabled(ds != NULL);
if (ds->b[core_id] == NULL) {
return LIB_ERR_NO_SPANNED_DISP;
}
thread_enqueue(thread, &ds->remote_wakeup_queue);
// Signal the inter-disp waitset of this event
struct event_closure closure = {
.handler = handle_wakeup_on
};
errval_t err =
waitset_chan_trigger_closure_disabled(&ds->interdisp_ws,
&ds->remote_wakeup_event,
closure,
mydisp);
assert_disabled(err_is_ok(err) ||
err_no(err) == LIB_ERR_CHAN_ALREADY_REGISTERED);
return SYS_ERR_OK;
}
/// Returns a channel with a pending event on the given waitset, or NULL
static struct waitset_chanstate *get_pending_event_disabled(struct waitset *ws)
{
// are there any pending events on the waitset?
if (ws->pending == NULL) {
return NULL;
}
// dequeue next pending event
struct waitset_chanstate *chan = ws->pending;
if (chan->next == chan) {
assert_disabled(chan->prev == chan);
ws->pending = NULL;
} else {
ws->pending = chan->next;
chan->prev->next = chan->next;
chan->next->prev = chan->prev;
}
#ifndef NDEBUG
chan->prev = chan->next = NULL;
#endif
// mark not pending
assert_disabled(chan->state == CHAN_PENDING);
chan->state = CHAN_UNREGISTERED;
chan->waitset = NULL;
return chan;
}
/**
* \brief Register a closure to be called when a channel is triggered
*
* In the Future, call the closure on a thread associated with the waitset
* when the channel is triggered. Only one closure may be registered per
* channel state at any one time.
* This function must only be called when disabled.
*
* \param ws Waitset
* \param chan Waitset's per-channel state
* \param closure Event handler
*/
errval_t waitset_chan_register_disabled(struct waitset *ws,
struct waitset_chanstate *chan,
struct event_closure closure)
{
if (chan->waitset != NULL) {
return LIB_ERR_CHAN_ALREADY_REGISTERED;
}
chan->waitset = ws;
// channel must not already be registered!
assert_disabled(chan->next == NULL && chan->prev == NULL);
assert_disabled(chan->state == CHAN_UNREGISTERED);
// this is probably insane! :)
assert_disabled(closure.handler != NULL);
// store closure
chan->closure = closure;
// enqueue this channel on the waitset's queue of idle channels
if (ws->idle == NULL) {
chan->next = chan->prev = chan;
ws->idle = chan;
} else {
chan->next = ws->idle;
chan->prev = chan->next->prev;
chan->next->prev = chan;
chan->prev->next = chan;
}
chan->state = CHAN_IDLE;
return SYS_ERR_OK;
}
/**
* \brief Resume execution of a given register state
*
* This function resumes the execution of the given register state on the
* current dispatcher. It may only be called while the dispatcher is disabled.
*
* \param disp Current dispatcher pointer
* \param regs Register state snapshot
*/
void
disp_resume(dispatcher_handle_t handle,
arch_registers_state_t *archregs)
{
struct dispatcher_shared_arm *disp =
get_dispatcher_shared_arm(handle);
// The definition of disp_resume_end is a totally flakey. The system
// uses the location of the PC to determine where to spill the thread
// context for exceptions and interrupts. There are two safe ways of doing
// this:
//
// 1) Write this entire function in assmebler.
// 2) Write this function in C and write a linker script to emit
// function bounds.
assert_disabled(curdispatcher() == handle);
assert_disabled(disp->d.disabled);
assert_disabled(disp->d.haswork);
#ifdef CONFIG_DEBUG_DEADLOCKS
((struct disp_priv *)disp)->yieldcount = 0;
#endif
disp_resume_context(&disp->d, archregs->regs);
}
/**
* \brief Save the current register state and optionally yield the CPU
*
* This function saves as much as necessary of the current register state
* (which, when resumed will return to the caller), and then either
* re-enters the thread scheduler or yields the CPU.
* It may only be called while the dispatcher is disabled.
*
* \param disp Current dispatcher pointer
* \param regs Location to save current register state
* \param yield If true, yield CPU to kernel; otherwise re-run thread scheduler
* \param yield_to Endpoint capability for dispatcher to which we want to yield
*/
void disp_save(dispatcher_handle_t handle,
arch_registers_state_t *state,
bool yield, capaddr_t yield_to)
{
struct dispatcher_shared_arm *disp =
get_dispatcher_shared_arm(handle);
assert_disabled(curdispatcher() == handle);
assert_disabled(disp->d.disabled);
disp_save_context(state->regs);
state->named.pc = (lvaddr_t)disp_save_epilog;
if (yield) {
sys_yield(yield_to);
// may fail if target doesn't exist; if so, just fall through
}
// this code won't run if the yield succeeded
// enter thread scheduler again
// this doesn't return, and will call disp_yield if there's nothing to do
thread_run_disabled(handle);
__asm volatile("disp_save_epilog:");
}
/**
* \brief Cancel an event registration made with lmp_chan_register_send()
*
* \param lc LMP channel
*/
errval_t lmp_chan_deregister_send(struct lmp_chan *lc)
{
assert(lc != NULL);
errval_t err = waitset_chan_deregister(&lc->send_waitset);
if (err_is_fail(err)) {
return err;
}
// dequeue from list of channels with send events
assert(lc->next != NULL && lc->prev != NULL);
dispatcher_handle_t handle = disp_disable();
struct dispatcher_generic *dp = get_dispatcher_generic(handle);
if (lc->next == lc->prev) {
assert_disabled(dp->lmp_send_events_list == lc);
dp->lmp_send_events_list = NULL;
} else {
lc->prev->next = lc->next;
lc->next->prev = lc->prev;
if (dp->lmp_send_events_list == lc) {
dp->lmp_send_events_list = lc->next;
}
}
#ifndef NDEBUG
lc->prev = lc->next = NULL;
#endif
disp_enable(handle);
return err;
}
/**
* \brief Switch execution between two register states
*
* This function saves as much as necessary of the current register state
* (which, when resumed will return to the caller), and switches execution
* by resuming the given register state. It may only be called while the
* dispatcher is disabled.
*
* \param disp Current dispatcher pointer
* \param from_regs Location to save current register state
* \param to_regs Location from which to resume new register state
*/
void disp_switch(dispatcher_handle_t handle,
arch_registers_state_t *from_state,
arch_registers_state_t *to_state)
{
struct dispatcher_shared_arm *disp =
get_dispatcher_shared_arm(handle);
assert_disabled(curdispatcher() == handle);
assert_disabled(disp->d.disabled);
assert_disabled(disp->d.haswork);
assert_disabled(to_state != NULL);
disp_save_context(from_state->regs);
from_state->named.pc = (lvaddr_t)disp_switch_epilog;
disp_resume_context(&disp->d, to_state->regs);
__asm volatile("disp_switch_epilog:");
}
/**
* \brief Trigger a specific event callback on an unregistered channel
*
* This function is equivalent to waitset_chan_register_disabled() immediately
* followed by waitset_chan_trigger_disabled(), but avoids unneccessary queue
* manipulation. This function must only be called when disabled.
*
* \param ws Waitset
* \param chan Waitset's per-channel state
* \param closure Event handler
* \param disp Current dispatcher pointer
*/
errval_t waitset_chan_trigger_closure_disabled(struct waitset *ws,
struct waitset_chanstate *chan,
struct event_closure closure,
dispatcher_handle_t handle)
{
assert_disabled(chan != NULL);
assert_disabled(ws != NULL);
// check if already registered
if (chan->waitset != NULL || chan->state != CHAN_UNREGISTERED) {
return LIB_ERR_CHAN_ALREADY_REGISTERED;
}
assert_disabled(chan->prev == NULL && chan->next == NULL);
// set closure
chan->closure = closure;
// is there a thread blocked on this waitset? if so, awaken it with the event
if (ws->waiting_threads != NULL) {
struct thread *t;
t = thread_unblock_one_disabled(handle, &ws->waiting_threads, chan);
assert_disabled(t == NULL);
return SYS_ERR_OK;
}
// mark channel pending and place on end of pending event queue
chan->waitset = ws;
chan->state = CHAN_PENDING;
if (ws->pending == NULL) {
ws->pending = chan;
chan->next = chan->prev = chan;
} else {
chan->next = ws->pending;
chan->prev = ws->pending->prev;
chan->next->prev = chan;
chan->prev->next = chan;
}
assert(ws->pending->prev != NULL && ws->pending->next != NULL);
return SYS_ERR_OK;
}
errval_t domain_wakeup_on_disabled(dispatcher_handle_t disp,
struct thread *thread,
dispatcher_handle_t mydisp)
{
coreid_t core_id = disp_handle_get_core_id(disp);
// TODO: Can't wakeup on anyone else than the owning dispatcher yet
assert_disabled(disp == thread->disp);
return domain_wakeup_on_coreid_disabled(core_id, thread, mydisp);
}
static void wakeup_thread_request(struct interdisp_binding *b,
genvaddr_t taddr)
{
coreid_t core_id = disp_get_core_id();
struct thread *wakeup = (struct thread *)(uintptr_t)taddr;
dispatcher_handle_t handle = disp_disable();
struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle);
/* assert_disabled(wakeup->disp == handle); */
assert_disabled(wakeup->coreid == core_id);
wakeup->disp = handle;
thread_enqueue(wakeup, &disp_gen->runq);
disp_enable(handle);
}
/**
* \brief Register a closure on a channel, and mark the channel as polled
*
* In the Future, call the closure on a thread associated with the waitset
* when the channel is triggered. Only one closure may be registered per
* channel state at any one time. Additionally, mark the channel as polled.
* This function must only be called when disabled.
*
* \param ws Waitset
* \param chan Waitset's per-channel state
* \param closure Event handler
* \param disp Current dispatcher pointer
*/
errval_t waitset_chan_register_polled_disabled(struct waitset *ws,
struct waitset_chanstate *chan,
struct event_closure closure,
dispatcher_handle_t handle)
{
if (chan->waitset != NULL) {
return LIB_ERR_CHAN_ALREADY_REGISTERED;
}
chan->waitset = ws;
// channel must not already be registered!
assert_disabled(chan->next == NULL && chan->prev == NULL);
assert_disabled(chan->state == CHAN_UNREGISTERED);
// store closure
chan->closure = closure;
// enqueue this channel on the waitset's queue of polled channels
if (ws->polled == NULL) {
chan->next = chan->prev = chan;
ws->polled = chan;
if (ws->waiting_threads != NULL && !ws->polling) {
// start a blocked thread polling
ws->polling = true;
struct thread *t;
t = thread_unblock_one_disabled(handle, &ws->waiting_threads, NULL);
assert_disabled(t == NULL); // shouldn't see a remote thread: waitsets are per-dispatcher
}
} else {
chan->next = ws->polled;
chan->prev = chan->next->prev;
chan->next->prev = chan;
chan->prev->next = chan;
}
chan->state = CHAN_POLLED;
return SYS_ERR_OK;
}
/**
* \brief Trigger send events for all LMP channels that are registered
*
* We don't have a good way to determine when we are likely to be able
* to send on an LMP channel, so this function just trigger all such
* pending events every time the dispatcher is rescheduled.
*
* Must be called while disabled and from dispatcher logic.
*/
void lmp_channels_retry_send_disabled(dispatcher_handle_t handle)
{
struct dispatcher_generic *dp = get_dispatcher_generic(handle);
struct lmp_chan *lc, *first = dp->lmp_send_events_list, *next;
errval_t err;
for (lc = first; lc != NULL; lc = next) {
next = lc->next;
assert(next != NULL);
err = waitset_chan_trigger_disabled(&lc->send_waitset, handle);
assert_disabled(err_is_ok(err)); // shouldn't fail
#ifndef NDEBUG
lc->next = lc->prev = NULL;
#endif
if (next == first) {
break; // wrapped
}
}
dp->lmp_send_events_list = NULL;
}
/**
* \brief Trigger an event callback on a channel
*
* Marks the given channel as having a pending event, causing some future call
* to get_next_event() to return the registered closure.
* This function must only be called when disabled.
*
* \param chan Waitset's per-channel state
* \param disp Current dispatcher pointer
*/
errval_t waitset_chan_trigger_disabled(struct waitset_chanstate *chan,
dispatcher_handle_t handle)
{
assert_disabled(chan != NULL);
struct waitset *ws = chan->waitset;
assert_disabled(ws != NULL);
assert_disabled(chan->prev != NULL && chan->next != NULL);
// no-op if already pending
if (chan->state == CHAN_PENDING) {
return SYS_ERR_OK;
}
// remove from previous queue (either idle or polled)
if (chan->next == chan) {
assert_disabled(chan->prev == chan);
if (chan->state == CHAN_IDLE) {
assert_disabled(ws->idle == chan);
ws->idle = NULL;
} else {
assert_disabled(chan->state == CHAN_POLLED);
assert_disabled(ws->polled == chan);
ws->polled = NULL;
}
} else {
chan->prev->next = chan->next;
chan->next->prev = chan->prev;
if (chan->state == CHAN_IDLE) {
if (ws->idle == chan) {
ws->idle = chan->next;
}
} else {
assert_disabled(chan->state == CHAN_POLLED);
if (ws->polled == chan) {
ws->polled = chan->next;
}
}
}
// is there a thread blocked on this waitset? if so, awaken it with the event
if (ws->waiting_threads != NULL) {
chan->waitset = NULL;
#ifndef NDEBUG
chan->prev = chan->next = NULL;
#endif
chan->state = CHAN_UNREGISTERED;
struct thread *t;
t = thread_unblock_one_disabled(handle, &ws->waiting_threads, chan);
assert_disabled(t == NULL);
return SYS_ERR_OK;
}
// else mark channel pending and move to end of pending event queue
chan->state = CHAN_PENDING;
if (ws->pending == NULL) {
ws->pending = chan;
chan->next = chan->prev = chan;
} else {
chan->next = ws->pending;
chan->prev = ws->pending->prev;
assert_disabled(ws->pending->next != NULL);
assert_disabled(ws->pending->prev != NULL);
assert_disabled(chan->prev != NULL);
chan->next->prev = chan;
chan->prev->next = chan;
}
return SYS_ERR_OK;
}
/**
* \brief Cancel a previous callback registration
*
* Remove the registration for a callback on the given channel.
* This function must only be called when disabled.
*
* \param chan Waitset's per-channel state
*/
errval_t waitset_chan_deregister_disabled(struct waitset_chanstate *chan)
{
assert_disabled(chan != NULL);
struct waitset *ws = chan->waitset;
if (ws == NULL) {
return LIB_ERR_CHAN_NOT_REGISTERED;
}
// remove this channel from the queue in which it is waiting
chan->waitset = NULL;
assert_disabled(chan->next != NULL && chan->prev != NULL);
if (chan->next == chan) {
// only thing in the list: must be the head
assert_disabled(chan->prev == chan);
switch (chan->state) {
case CHAN_IDLE:
assert_disabled(chan == ws->idle);
ws->idle = NULL;
break;
case CHAN_POLLED:
assert_disabled(chan == ws->polled);
ws->polled = NULL;
break;
case CHAN_PENDING:
assert_disabled(chan == ws->pending);
ws->pending = NULL;
break;
default:
assert_disabled(!"invalid channel state in deregister");
}
} else {
assert_disabled(chan->prev != chan);
chan->prev->next = chan->next;
chan->next->prev = chan->prev;
switch (chan->state) {
case CHAN_IDLE:
if (chan == ws->idle) {
ws->idle = chan->next;
}
break;
case CHAN_POLLED:
if (chan == ws->polled) {
ws->polled = chan->next;
}
break;
case CHAN_PENDING:
if (chan == ws->pending) {
ws->pending = chan->next;
}
break;
default:
assert_disabled(!"invalid channel state in deregister");
}
}
chan->state = CHAN_UNREGISTERED;
#ifndef NDEBUG
chan->prev = chan->next = NULL;
#endif
return SYS_ERR_OK;
}
static errval_t get_next_event_debug(struct waitset *ws,
struct event_closure *retclosure, bool debug)
{
struct waitset_chanstate *chan;
bool was_polling = false;
cycles_t pollcycles;
assert(ws != NULL);
assert(retclosure != NULL);
// unconditionally disable ourselves and check for events
// if we decide we have to start polling, we'll jump back up here
goto check_for_events;
/* ------------ POLLING LOOP; RUNS WHILE ENABLED ------------ */
polling_loop:
was_polling = true;
assert(ws->polling); // this thread is polling
// get the amount of cycles we want to poll for
pollcycles = pollcycles_reset();
// while there are no pending events, poll channels
while (ws->polled != NULL && ws->pending == NULL) {
struct waitset_chanstate *nextchan = NULL;
// NB: Polling policy is to return as soon as a pending event
// appears, not bother looking at the rest of the polling queue
for (chan = ws->polled;
chan != NULL && chan->waitset == ws && chan->state == CHAN_POLLED
&& ws->pending == NULL;
chan = nextchan) {
nextchan = chan->next;
poll_channel(chan);
// update pollcycles
pollcycles = pollcycles_update(pollcycles);
// yield the thread if we exceed the cycle count limit
if (ws->pending == NULL && pollcycles_expired(pollcycles)) {
if (debug) {
if (strcmp(disp_name(), "netd") != 0) {
// Print the callback trace so that we know which call is leading
// the schedule removal and
printf("%s: callstack: %p %p %p %p\n", disp_name(),
__builtin_return_address(0),
__builtin_return_address(1),
__builtin_return_address(2),
__builtin_return_address(3));
}
}
thread_yield();
pollcycles = pollcycles_reset();
}
}
// ensure that we restart polling from the place we left off here,
// if the next channel is a valid one
if (nextchan != NULL && nextchan->waitset == ws
&& nextchan->state == CHAN_POLLED) {
ws->polled = nextchan;
}
}
/* ------------ STATE MACHINERY; RUNS WHILE DISABLED ------------ */
check_for_events: ;
dispatcher_handle_t handle = disp_disable();
// are there any pending events on the waitset?
chan = get_pending_event_disabled(ws);
if (chan != NULL) {
// if we need to poll, and we have a blocked thread, wake it up to do so
if (was_polling && ws->polled != NULL && ws->waiting_threads != NULL) {
// start a blocked thread polling
struct thread *t;
t = thread_unblock_one_disabled(handle, &ws->waiting_threads, NULL);
assert_disabled(t == NULL); // shouldn't see a remote thread
} else if (was_polling) {
// I'm stopping polling, and there is nobody else
assert_disabled(ws->polling);
ws->polling = false;
}
disp_enable(handle);
*retclosure = chan->closure;
return SYS_ERR_OK;
}
// If we got here and there are channels to poll but no-one is polling,
// then either we never polled, or we lost a race on the channel we picked.
// Either way, we'd better start polling again.
if (ws->polled != NULL && (was_polling || !ws->polling)) {
if (!was_polling) {
ws->polling = true;
}
disp_enable(handle);
goto polling_loop;
}
// otherwise block awaiting an event
chan = thread_block_disabled(handle, &ws->waiting_threads);
if (chan == NULL) {
// not a real event, just a wakeup to get us to start polling!
assert(ws->polling);
goto polling_loop;
} else {
*retclosure = chan->closure;
return SYS_ERR_OK;
}
}