/*
* pserialize_switchpoint:
*
* Monitor system context switch activity. Called from machine
* independent code after mi_switch() returns.
*/
void
pserialize_switchpoint(void)
{
pserialize_t psz, next;
cpuid_t cid;
/*
* If no updates pending, bail out. No need to lock in order to
* test psz_work_todo; the only ill effect of missing an update
* would be to delay LWPs waiting in pserialize_perform(). That
* will not happen because updates are on the queue before an
* xcall is generated (serialization) to tickle every CPU.
*/
if (__predict_true(psz_work_todo == 0)) {
return;
}
mutex_spin_enter(&psz_lock);
cid = cpu_index(curcpu());
/*
* At first, scan through the second queue and update each request,
* if passed all processors, then transfer to the third queue.
*/
for (psz = TAILQ_FIRST(&psz_queue1); psz != NULL; psz = next) {
next = TAILQ_NEXT(psz, psz_chain);
kcpuset_set(psz->psz_pass, cid);
if (!kcpuset_match(psz->psz_pass, psz->psz_target)) {
continue;
}
kcpuset_zero(psz->psz_pass);
TAILQ_REMOVE(&psz_queue1, psz, psz_chain);
TAILQ_INSERT_TAIL(&psz_queue2, psz, psz_chain);
}
/*
* Scan through the first queue and update each request,
* if passed all processors, then move to the second queue.
*/
for (psz = TAILQ_FIRST(&psz_queue0); psz != NULL; psz = next) {
next = TAILQ_NEXT(psz, psz_chain);
kcpuset_set(psz->psz_pass, cid);
if (!kcpuset_match(psz->psz_pass, psz->psz_target)) {
continue;
}
kcpuset_zero(psz->psz_pass);
TAILQ_REMOVE(&psz_queue0, psz, psz_chain);
TAILQ_INSERT_TAIL(&psz_queue1, psz, psz_chain);
}
/*
* Process the third queue: entries have been seen twice on every
* processor, remove from the queue and notify the updating thread.
*/
while ((psz = TAILQ_FIRST(&psz_queue2)) != NULL) {
TAILQ_REMOVE(&psz_queue2, psz, psz_chain);
kcpuset_zero(psz->psz_target);
psz_work_todo--;
}
mutex_spin_exit(&psz_lock);
}
/*
* Halt all running cpus, excluding current cpu.
*/
void
cpu_halt_others(void)
{
kcpuset_t *kcp;
// If we are the only CPU running, there's nothing to do.
if (kcpuset_match(cpus_running, curcpu()->ci_data.cpu_kcpuset))
return;
// Get all running CPUs
kcpuset_clone(&kcp, cpus_running);
// Remove ourself
kcpuset_remove(kcp, curcpu()->ci_data.cpu_kcpuset);
// Remove any halted CPUs
kcpuset_remove(kcp, cpus_halted);
// If there are CPUs left, send the IPIs
if (!kcpuset_iszero(kcp)) {
cpu_multicast_ipi(kcp, IPI_HALT);
cpu_ipi_wait("halt", cpus_halted, kcp);
}
kcpuset_destroy(kcp);
/*
* TBD
* Depending on available firmware methods, other cpus will
* either shut down themselves, or spin and wait for us to
* stop them.
*/
}
static void
cpu_ipi_wait(const char *s, const kcpuset_t *watchset, const kcpuset_t *wanted)
{
bool done = false;
struct cpu_info * const ci = curcpu();
kcpuset_t *kcp = ci->ci_watchcpus;
/* some finite amount of time */
for (u_long limit = curcpu()->ci_cpu_freq/10; !done && limit--; ) {
kcpuset_copy(kcp, watchset);
kcpuset_intersect(kcp, wanted);
done = kcpuset_match(kcp, wanted);
}
if (!done) {
cpuid_t cii;
kcpuset_copy(kcp, wanted);
kcpuset_remove(kcp, watchset);
if ((cii = kcpuset_ffs(kcp)) != 0) {
printf("Failed to %s:", s);
do {
kcpuset_clear(kcp, --cii);
printf(" cpu%lu", cii);
} while ((cii = kcpuset_ffs(kcp)) != 0);
printf("\n");
}
}
}
/*
* Pause all running cpus, excluding current cpu.
*/
void
cpu_pause_others(void)
{
struct cpu_info * const ci = curcpu();
if (cold || kcpuset_match(cpus_running, ci->ci_data.cpu_kcpuset))
return;
kcpuset_t *kcp = ci->ci_ddbcpus;
kcpuset_copy(kcp, cpus_running);
kcpuset_remove(kcp, ci->ci_data.cpu_kcpuset);
kcpuset_remove(kcp, cpus_paused);
cpu_broadcast_ipi(IPI_SUSPEND);
cpu_ipi_wait("pause", cpus_paused, kcp);
}
void
cpu_multicast_ipi(const kcpuset_t *kcp, int tag)
{
struct cpu_info * const ci = curcpu();
kcpuset_t *kcp2 = ci->ci_multicastcpus;
if (kcpuset_match(cpus_running, ci->ci_data.cpu_kcpuset))
return;
kcpuset_copy(kcp2, kcp);
kcpuset_remove(kcp2, ci->ci_data.cpu_kcpuset);
for (cpuid_t cii; (cii = kcpuset_ffs(kcp2)) != 0; ) {
kcpuset_clear(kcp2, --cii);
(void)cpu_send_ipi(cpu_lookup(cii), tag);
}
}
static inline void
pmap_tlb_processpacket(pmap_tlb_packet_t *tp, kcpuset_t *target)
{
int err = 0;
if (!kcpuset_match(target, kcpuset_attached)) {
const struct cpu_info * const self = curcpu();
CPU_INFO_ITERATOR cii;
struct cpu_info *lci;
for (CPU_INFO_FOREACH(cii, lci)) {
const cpuid_t lcid = cpu_index(lci);
if (__predict_false(lci == self) ||
!kcpuset_isset(target, lcid)) {
continue;
}
err |= x86_ipi(LAPIC_TLB_VECTOR,
lci->ci_cpuid, LAPIC_DLMODE_FIXED);
}
} else {
