/// Destroy the local state associated with a given channel void lmp_chan_destroy(struct lmp_chan *lc) { lc->connstate = LMP_DISCONNECTED; cap_destroy(lc->local_cap); if (lc->endpoint != NULL) { lmp_endpoint_free(lc->endpoint); } // remove from send retry queue on dispatcher if (waitset_chan_is_registered(&lc->send_waitset)) { assert(lc->prev != NULL && lc->next != NULL); dispatcher_handle_t handle = disp_disable(); struct dispatcher_generic *disp = get_dispatcher_generic(handle); if (lc->next == lc->prev) { assert_disabled(lc->next == lc); assert_disabled(disp->lmp_send_events_list == lc); disp->lmp_send_events_list = NULL; } else { lc->prev->next = lc->next; lc->next->prev = lc->prev; } disp_enable(handle); #ifndef NDEBUG lc->next = lc->prev = NULL; #endif } waitset_chanstate_destroy(&lc->send_waitset); }
/** * \brief Wakeup a thread on a foreign dispatcher while disabled. * * \param core_id Core ID to wakeup on * \param thread Pointer to thread to wakeup * \param mydisp Dispatcher this function is running on * * \return SYS_ERR_OK on success. */ static errval_t domain_wakeup_on_coreid_disabled(coreid_t core_id, struct thread *thread, dispatcher_handle_t mydisp) { struct domain_state *ds = get_domain_state(); // XXX: Ugly hack to allow waking up on a core id we don't have a // dispatcher handler for thread->coreid = core_id; // Catch this early assert_disabled(ds != NULL); if (ds->b[core_id] == NULL) { return LIB_ERR_NO_SPANNED_DISP; } thread_enqueue(thread, &ds->remote_wakeup_queue); // Signal the inter-disp waitset of this event struct event_closure closure = { .handler = handle_wakeup_on }; errval_t err = waitset_chan_trigger_closure_disabled(&ds->interdisp_ws, &ds->remote_wakeup_event, closure, mydisp); assert_disabled(err_is_ok(err) || err_no(err) == LIB_ERR_CHAN_ALREADY_REGISTERED); return SYS_ERR_OK; }
/// Returns a channel with a pending event on the given waitset, or NULL static struct waitset_chanstate *get_pending_event_disabled(struct waitset *ws) { // are there any pending events on the waitset? if (ws->pending == NULL) { return NULL; } // dequeue next pending event struct waitset_chanstate *chan = ws->pending; if (chan->next == chan) { assert_disabled(chan->prev == chan); ws->pending = NULL; } else { ws->pending = chan->next; chan->prev->next = chan->next; chan->next->prev = chan->prev; } #ifndef NDEBUG chan->prev = chan->next = NULL; #endif // mark not pending assert_disabled(chan->state == CHAN_PENDING); chan->state = CHAN_UNREGISTERED; chan->waitset = NULL; return chan; }
/** * \brief Register a closure to be called when a channel is triggered * * In the Future, call the closure on a thread associated with the waitset * when the channel is triggered. Only one closure may be registered per * channel state at any one time. * This function must only be called when disabled. * * \param ws Waitset * \param chan Waitset's per-channel state * \param closure Event handler */ errval_t waitset_chan_register_disabled(struct waitset *ws, struct waitset_chanstate *chan, struct event_closure closure) { if (chan->waitset != NULL) { return LIB_ERR_CHAN_ALREADY_REGISTERED; } chan->waitset = ws; // channel must not already be registered! assert_disabled(chan->next == NULL && chan->prev == NULL); assert_disabled(chan->state == CHAN_UNREGISTERED); // this is probably insane! :) assert_disabled(closure.handler != NULL); // store closure chan->closure = closure; // enqueue this channel on the waitset's queue of idle channels if (ws->idle == NULL) { chan->next = chan->prev = chan; ws->idle = chan; } else { chan->next = ws->idle; chan->prev = chan->next->prev; chan->next->prev = chan; chan->prev->next = chan; } chan->state = CHAN_IDLE; return SYS_ERR_OK; }
/** * \brief Resume execution of a given register state * * This function resumes the execution of the given register state on the * current dispatcher. It may only be called while the dispatcher is disabled. * * \param disp Current dispatcher pointer * \param regs Register state snapshot */ void disp_resume(dispatcher_handle_t handle, arch_registers_state_t *archregs) { struct dispatcher_shared_arm *disp = get_dispatcher_shared_arm(handle); // The definition of disp_resume_end is a totally flakey. The system // uses the location of the PC to determine where to spill the thread // context for exceptions and interrupts. There are two safe ways of doing // this: // // 1) Write this entire function in assmebler. // 2) Write this function in C and write a linker script to emit // function bounds. assert_disabled(curdispatcher() == handle); assert_disabled(disp->d.disabled); assert_disabled(disp->d.haswork); #ifdef CONFIG_DEBUG_DEADLOCKS ((struct disp_priv *)disp)->yieldcount = 0; #endif disp_resume_context(&disp->d, archregs->regs); }
/** * \brief Save the current register state and optionally yield the CPU * * This function saves as much as necessary of the current register state * (which, when resumed will return to the caller), and then either * re-enters the thread scheduler or yields the CPU. * It may only be called while the dispatcher is disabled. * * \param disp Current dispatcher pointer * \param regs Location to save current register state * \param yield If true, yield CPU to kernel; otherwise re-run thread scheduler * \param yield_to Endpoint capability for dispatcher to which we want to yield */ void disp_save(dispatcher_handle_t handle, arch_registers_state_t *state, bool yield, capaddr_t yield_to) { struct dispatcher_shared_arm *disp = get_dispatcher_shared_arm(handle); assert_disabled(curdispatcher() == handle); assert_disabled(disp->d.disabled); disp_save_context(state->regs); state->named.pc = (lvaddr_t)disp_save_epilog; if (yield) { sys_yield(yield_to); // may fail if target doesn't exist; if so, just fall through } // this code won't run if the yield succeeded // enter thread scheduler again // this doesn't return, and will call disp_yield if there's nothing to do thread_run_disabled(handle); __asm volatile("disp_save_epilog:"); }
/** * \brief Cancel an event registration made with lmp_chan_register_send() * * \param lc LMP channel */ errval_t lmp_chan_deregister_send(struct lmp_chan *lc) { assert(lc != NULL); errval_t err = waitset_chan_deregister(&lc->send_waitset); if (err_is_fail(err)) { return err; } // dequeue from list of channels with send events assert(lc->next != NULL && lc->prev != NULL); dispatcher_handle_t handle = disp_disable(); struct dispatcher_generic *dp = get_dispatcher_generic(handle); if (lc->next == lc->prev) { assert_disabled(dp->lmp_send_events_list == lc); dp->lmp_send_events_list = NULL; } else { lc->prev->next = lc->next; lc->next->prev = lc->prev; if (dp->lmp_send_events_list == lc) { dp->lmp_send_events_list = lc->next; } } #ifndef NDEBUG lc->prev = lc->next = NULL; #endif disp_enable(handle); return err; }
/** * \brief Switch execution between two register states * * This function saves as much as necessary of the current register state * (which, when resumed will return to the caller), and switches execution * by resuming the given register state. It may only be called while the * dispatcher is disabled. * * \param disp Current dispatcher pointer * \param from_regs Location to save current register state * \param to_regs Location from which to resume new register state */ void disp_switch(dispatcher_handle_t handle, arch_registers_state_t *from_state, arch_registers_state_t *to_state) { struct dispatcher_shared_arm *disp = get_dispatcher_shared_arm(handle); assert_disabled(curdispatcher() == handle); assert_disabled(disp->d.disabled); assert_disabled(disp->d.haswork); assert_disabled(to_state != NULL); disp_save_context(from_state->regs); from_state->named.pc = (lvaddr_t)disp_switch_epilog; disp_resume_context(&disp->d, to_state->regs); __asm volatile("disp_switch_epilog:"); }
/** * \brief Trigger a specific event callback on an unregistered channel * * This function is equivalent to waitset_chan_register_disabled() immediately * followed by waitset_chan_trigger_disabled(), but avoids unneccessary queue * manipulation. This function must only be called when disabled. * * \param ws Waitset * \param chan Waitset's per-channel state * \param closure Event handler * \param disp Current dispatcher pointer */ errval_t waitset_chan_trigger_closure_disabled(struct waitset *ws, struct waitset_chanstate *chan, struct event_closure closure, dispatcher_handle_t handle) { assert_disabled(chan != NULL); assert_disabled(ws != NULL); // check if already registered if (chan->waitset != NULL || chan->state != CHAN_UNREGISTERED) { return LIB_ERR_CHAN_ALREADY_REGISTERED; } assert_disabled(chan->prev == NULL && chan->next == NULL); // set closure chan->closure = closure; // is there a thread blocked on this waitset? if so, awaken it with the event if (ws->waiting_threads != NULL) { struct thread *t; t = thread_unblock_one_disabled(handle, &ws->waiting_threads, chan); assert_disabled(t == NULL); return SYS_ERR_OK; } // mark channel pending and place on end of pending event queue chan->waitset = ws; chan->state = CHAN_PENDING; if (ws->pending == NULL) { ws->pending = chan; chan->next = chan->prev = chan; } else { chan->next = ws->pending; chan->prev = ws->pending->prev; chan->next->prev = chan; chan->prev->next = chan; } assert(ws->pending->prev != NULL && ws->pending->next != NULL); return SYS_ERR_OK; }
errval_t domain_wakeup_on_disabled(dispatcher_handle_t disp, struct thread *thread, dispatcher_handle_t mydisp) { coreid_t core_id = disp_handle_get_core_id(disp); // TODO: Can't wakeup on anyone else than the owning dispatcher yet assert_disabled(disp == thread->disp); return domain_wakeup_on_coreid_disabled(core_id, thread, mydisp); }
static void wakeup_thread_request(struct interdisp_binding *b, genvaddr_t taddr) { coreid_t core_id = disp_get_core_id(); struct thread *wakeup = (struct thread *)(uintptr_t)taddr; dispatcher_handle_t handle = disp_disable(); struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle); /* assert_disabled(wakeup->disp == handle); */ assert_disabled(wakeup->coreid == core_id); wakeup->disp = handle; thread_enqueue(wakeup, &disp_gen->runq); disp_enable(handle); }
/** * \brief Register a closure on a channel, and mark the channel as polled * * In the Future, call the closure on a thread associated with the waitset * when the channel is triggered. Only one closure may be registered per * channel state at any one time. Additionally, mark the channel as polled. * This function must only be called when disabled. * * \param ws Waitset * \param chan Waitset's per-channel state * \param closure Event handler * \param disp Current dispatcher pointer */ errval_t waitset_chan_register_polled_disabled(struct waitset *ws, struct waitset_chanstate *chan, struct event_closure closure, dispatcher_handle_t handle) { if (chan->waitset != NULL) { return LIB_ERR_CHAN_ALREADY_REGISTERED; } chan->waitset = ws; // channel must not already be registered! assert_disabled(chan->next == NULL && chan->prev == NULL); assert_disabled(chan->state == CHAN_UNREGISTERED); // store closure chan->closure = closure; // enqueue this channel on the waitset's queue of polled channels if (ws->polled == NULL) { chan->next = chan->prev = chan; ws->polled = chan; if (ws->waiting_threads != NULL && !ws->polling) { // start a blocked thread polling ws->polling = true; struct thread *t; t = thread_unblock_one_disabled(handle, &ws->waiting_threads, NULL); assert_disabled(t == NULL); // shouldn't see a remote thread: waitsets are per-dispatcher } } else { chan->next = ws->polled; chan->prev = chan->next->prev; chan->next->prev = chan; chan->prev->next = chan; } chan->state = CHAN_POLLED; return SYS_ERR_OK; }
/** * \brief Trigger send events for all LMP channels that are registered * * We don't have a good way to determine when we are likely to be able * to send on an LMP channel, so this function just trigger all such * pending events every time the dispatcher is rescheduled. * * Must be called while disabled and from dispatcher logic. */ void lmp_channels_retry_send_disabled(dispatcher_handle_t handle) { struct dispatcher_generic *dp = get_dispatcher_generic(handle); struct lmp_chan *lc, *first = dp->lmp_send_events_list, *next; errval_t err; for (lc = first; lc != NULL; lc = next) { next = lc->next; assert(next != NULL); err = waitset_chan_trigger_disabled(&lc->send_waitset, handle); assert_disabled(err_is_ok(err)); // shouldn't fail #ifndef NDEBUG lc->next = lc->prev = NULL; #endif if (next == first) { break; // wrapped } } dp->lmp_send_events_list = NULL; }
/** * \brief Trigger an event callback on a channel * * Marks the given channel as having a pending event, causing some future call * to get_next_event() to return the registered closure. * This function must only be called when disabled. * * \param chan Waitset's per-channel state * \param disp Current dispatcher pointer */ errval_t waitset_chan_trigger_disabled(struct waitset_chanstate *chan, dispatcher_handle_t handle) { assert_disabled(chan != NULL); struct waitset *ws = chan->waitset; assert_disabled(ws != NULL); assert_disabled(chan->prev != NULL && chan->next != NULL); // no-op if already pending if (chan->state == CHAN_PENDING) { return SYS_ERR_OK; } // remove from previous queue (either idle or polled) if (chan->next == chan) { assert_disabled(chan->prev == chan); if (chan->state == CHAN_IDLE) { assert_disabled(ws->idle == chan); ws->idle = NULL; } else { assert_disabled(chan->state == CHAN_POLLED); assert_disabled(ws->polled == chan); ws->polled = NULL; } } else { chan->prev->next = chan->next; chan->next->prev = chan->prev; if (chan->state == CHAN_IDLE) { if (ws->idle == chan) { ws->idle = chan->next; } } else { assert_disabled(chan->state == CHAN_POLLED); if (ws->polled == chan) { ws->polled = chan->next; } } } // is there a thread blocked on this waitset? if so, awaken it with the event if (ws->waiting_threads != NULL) { chan->waitset = NULL; #ifndef NDEBUG chan->prev = chan->next = NULL; #endif chan->state = CHAN_UNREGISTERED; struct thread *t; t = thread_unblock_one_disabled(handle, &ws->waiting_threads, chan); assert_disabled(t == NULL); return SYS_ERR_OK; } // else mark channel pending and move to end of pending event queue chan->state = CHAN_PENDING; if (ws->pending == NULL) { ws->pending = chan; chan->next = chan->prev = chan; } else { chan->next = ws->pending; chan->prev = ws->pending->prev; assert_disabled(ws->pending->next != NULL); assert_disabled(ws->pending->prev != NULL); assert_disabled(chan->prev != NULL); chan->next->prev = chan; chan->prev->next = chan; } return SYS_ERR_OK; }
/** * \brief Cancel a previous callback registration * * Remove the registration for a callback on the given channel. * This function must only be called when disabled. * * \param chan Waitset's per-channel state */ errval_t waitset_chan_deregister_disabled(struct waitset_chanstate *chan) { assert_disabled(chan != NULL); struct waitset *ws = chan->waitset; if (ws == NULL) { return LIB_ERR_CHAN_NOT_REGISTERED; } // remove this channel from the queue in which it is waiting chan->waitset = NULL; assert_disabled(chan->next != NULL && chan->prev != NULL); if (chan->next == chan) { // only thing in the list: must be the head assert_disabled(chan->prev == chan); switch (chan->state) { case CHAN_IDLE: assert_disabled(chan == ws->idle); ws->idle = NULL; break; case CHAN_POLLED: assert_disabled(chan == ws->polled); ws->polled = NULL; break; case CHAN_PENDING: assert_disabled(chan == ws->pending); ws->pending = NULL; break; default: assert_disabled(!"invalid channel state in deregister"); } } else { assert_disabled(chan->prev != chan); chan->prev->next = chan->next; chan->next->prev = chan->prev; switch (chan->state) { case CHAN_IDLE: if (chan == ws->idle) { ws->idle = chan->next; } break; case CHAN_POLLED: if (chan == ws->polled) { ws->polled = chan->next; } break; case CHAN_PENDING: if (chan == ws->pending) { ws->pending = chan->next; } break; default: assert_disabled(!"invalid channel state in deregister"); } } chan->state = CHAN_UNREGISTERED; #ifndef NDEBUG chan->prev = chan->next = NULL; #endif return SYS_ERR_OK; }
static errval_t get_next_event_debug(struct waitset *ws, struct event_closure *retclosure, bool debug) { struct waitset_chanstate *chan; bool was_polling = false; cycles_t pollcycles; assert(ws != NULL); assert(retclosure != NULL); // unconditionally disable ourselves and check for events // if we decide we have to start polling, we'll jump back up here goto check_for_events; /* ------------ POLLING LOOP; RUNS WHILE ENABLED ------------ */ polling_loop: was_polling = true; assert(ws->polling); // this thread is polling // get the amount of cycles we want to poll for pollcycles = pollcycles_reset(); // while there are no pending events, poll channels while (ws->polled != NULL && ws->pending == NULL) { struct waitset_chanstate *nextchan = NULL; // NB: Polling policy is to return as soon as a pending event // appears, not bother looking at the rest of the polling queue for (chan = ws->polled; chan != NULL && chan->waitset == ws && chan->state == CHAN_POLLED && ws->pending == NULL; chan = nextchan) { nextchan = chan->next; poll_channel(chan); // update pollcycles pollcycles = pollcycles_update(pollcycles); // yield the thread if we exceed the cycle count limit if (ws->pending == NULL && pollcycles_expired(pollcycles)) { if (debug) { if (strcmp(disp_name(), "netd") != 0) { // Print the callback trace so that we know which call is leading // the schedule removal and printf("%s: callstack: %p %p %p %p\n", disp_name(), __builtin_return_address(0), __builtin_return_address(1), __builtin_return_address(2), __builtin_return_address(3)); } } thread_yield(); pollcycles = pollcycles_reset(); } } // ensure that we restart polling from the place we left off here, // if the next channel is a valid one if (nextchan != NULL && nextchan->waitset == ws && nextchan->state == CHAN_POLLED) { ws->polled = nextchan; } } /* ------------ STATE MACHINERY; RUNS WHILE DISABLED ------------ */ check_for_events: ; dispatcher_handle_t handle = disp_disable(); // are there any pending events on the waitset? chan = get_pending_event_disabled(ws); if (chan != NULL) { // if we need to poll, and we have a blocked thread, wake it up to do so if (was_polling && ws->polled != NULL && ws->waiting_threads != NULL) { // start a blocked thread polling struct thread *t; t = thread_unblock_one_disabled(handle, &ws->waiting_threads, NULL); assert_disabled(t == NULL); // shouldn't see a remote thread } else if (was_polling) { // I'm stopping polling, and there is nobody else assert_disabled(ws->polling); ws->polling = false; } disp_enable(handle); *retclosure = chan->closure; return SYS_ERR_OK; } // If we got here and there are channels to poll but no-one is polling, // then either we never polled, or we lost a race on the channel we picked. // Either way, we'd better start polling again. if (ws->polled != NULL && (was_polling || !ws->polling)) { if (!was_polling) { ws->polling = true; } disp_enable(handle); goto polling_loop; } // otherwise block awaiting an event chan = thread_block_disabled(handle, &ws->waiting_threads); if (chan == NULL) { // not a real event, just a wakeup to get us to start polling! assert(ws->polling); goto polling_loop; } else { *retclosure = chan->closure; return SYS_ERR_OK; } }