void
intr_disestablish(void *ih)
{
struct intrsource * const is = ih;
KASSERT(!cpu_intr_p());
KASSERT(!cpu_softintr_p());
pic_disestablish_source(is);
}
/*
* pcu_save_lwp: save PCU state to the given LWP.
*/
void
pcu_save(const pcu_ops_t *pcu)
{
const u_int id = pcu->pcu_id;
lwp_t * const l = curlwp;
KASSERT(!cpu_intr_p() && !cpu_softintr_p());
if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
return;
}
pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE);
}
/*
* pcu_load: load/initialize the PCU state of current LWP on current CPU.
*/
void
pcu_load(const pcu_ops_t *pcu)
{
const u_int id = pcu->pcu_id;
struct cpu_info *ci, *curci;
lwp_t * const l = curlwp;
uint64_t where;
int s;
KASSERT(!cpu_intr_p() && !cpu_softintr_p());
s = splsoftclock();
curci = curcpu();
ci = l->l_pcu_cpu[id];
/* Does this CPU already have our PCU state loaded? */
if (ci == curci) {
KASSERT(curci->ci_pcu_curlwp[id] == l);
pcu->pcu_state_load(l, PCU_ENABLE); /* Re-enable */
splx(s);
return;
}
/* If PCU state of this LWP is on the remote CPU - save it there. */
if (ci) {
splx(s);
/* Note: there is a race; see description in the top. */
where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op,
__UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci);
xc_wait(where);
/* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */
s = splsoftclock();
curci = curcpu();
}
KASSERT(l->l_pcu_cpu[id] == NULL);
/* Save the PCU state on the current CPU, if there is any. */
pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE);
KASSERT(curci->ci_pcu_curlwp[id] == NULL);
/*
* Finally, load the state for this LWP on this CPU. Indicate to
* load function whether PCU was used before. Note the usage.
*/
pcu_do_op(pcu, l, PCU_CLAIM | PCU_ENABLE | PCU_RELOAD);
splx(s);
}
/*
* pserialize_perform:
*
* Perform the write side of passive serialization. The calling
* thread holds an exclusive lock on the data object(s) being updated.
* We wait until every processor in the system has made at least two
* passes through cpu_swichto(). The wait is made with the caller's
* update lock held, but is short term.
*/
void
pserialize_perform(pserialize_t psz)
{
uint64_t xc;
KASSERT(!cpu_intr_p());
KASSERT(!cpu_softintr_p());
if (__predict_false(panicstr != NULL)) {
return;
}
KASSERT(psz->psz_owner == NULL);
KASSERT(ncpu > 0);
/*
* Set up the object and put it onto the queue. The lock
* activity here provides the necessary memory barrier to
* make the caller's data update completely visible to
* other processors.
*/
psz->psz_owner = curlwp;
kcpuset_copy(psz->psz_target, kcpuset_running);
kcpuset_zero(psz->psz_pass);
mutex_spin_enter(&psz_lock);
TAILQ_INSERT_TAIL(&psz_queue0, psz, psz_chain);
psz_work_todo++;
do {
mutex_spin_exit(&psz_lock);
/*
* Force some context switch activity on every CPU, as
* the system may not be busy. Pause to not flood.
*/
xc = xc_broadcast(XC_HIGHPRI, (xcfunc_t)nullop, NULL, NULL);
xc_wait(xc);
kpause("psrlz", false, 1, NULL);
mutex_spin_enter(&psz_lock);
} while (!kcpuset_iszero(psz->psz_target));
psz_ev_excl.ev_count++;
mutex_spin_exit(&psz_lock);
psz->psz_owner = NULL;
}
/*
* pcu_discard: discard the PCU state of current LWP.
* If the "usesw" flag is set, pcu_used_p() will return "true".
*/
void
pcu_discard(const pcu_ops_t *pcu, bool usesw)
{
const u_int id = pcu->pcu_id;
lwp_t * const l = curlwp;
KASSERT(!cpu_intr_p() && !cpu_softintr_p());
if (usesw)
l->l_pcu_used[PCU_USER] |= (1 << id);
else
l->l_pcu_used[PCU_USER] &= ~(1 << id);
if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
return;
}
pcu_lwp_op(pcu, l, PCU_RELEASE);
}
void *
intr_establish(int irq, int ipl, int type, int (*func)(void *), void *arg)
{
KASSERT(!cpu_intr_p());
KASSERT(!cpu_softintr_p());
for (size_t slot = 0; slot < PIC_MAXPICS; slot++) {
struct pic_softc * const pic = pic_list[slot];
if (pic == NULL || pic->pic_irqbase < 0)
continue;
if (pic->pic_irqbase <= irq
&& irq < pic->pic_irqbase + pic->pic_maxsources) {
return pic_establish_intr(pic, irq - pic->pic_irqbase,
ipl, type, func, arg);
}
}
return NULL;
}
