void
smp_rendezvous_cpus(unsigned long map,
void (* action_func)(void *),
void *arg)
{
unsigned int cpumask = 1 << cpu_number();
if (ncpus == 1) {
if (action_func != NULL)
action_func(arg);
return;
}
/* obtain rendezvous lock */
mtx_enter(&smp_ipi_mtx);
/* set static function pointers */
smp_rv_map = map;
smp_rv_action_func = action_func;
smp_rv_func_arg = arg;
smp_rv_waiters[0] = 0;
smp_rv_waiters[1] = 0;
/* signal other processors, which will enter the IPI with interrupts off */
mips64_multicast_ipi(map & ~cpumask, MIPS64_IPI_RENDEZVOUS);
/* Check if the current CPU is in the map */
if (map & cpumask)
smp_rendezvous_action();
while (smp_rv_waiters[1] != smp_rv_map)
;
/* release lock */
mtx_leave(&smp_ipi_mtx);
}
void
smp_rendezvous_cpus(cpumask_t map,
void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
int i, ncpus = 0;
if (!smp_started) {
if (setup_func != NULL)
setup_func(arg);
if (action_func != NULL)
action_func(arg);
if (teardown_func != NULL)
teardown_func(arg);
return;
}
CPU_FOREACH(i) {
if (((1 << i) & map) != 0)
ncpus++;
}
if (ncpus == 0)
panic("ncpus is 0 with map=0x%x", map);
/* obtain rendezvous lock */
mtx_lock_spin(&smp_ipi_mtx);
/* set static function pointers */
smp_rv_ncpus = ncpus;
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[1] = 0;
smp_rv_waiters[2] = 0;
atomic_store_rel_int(&smp_rv_waiters[0], 0);
/* signal other processors, which will enter the IPI with interrupts off */
ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS);
/* Check if the current CPU is in the map */
if ((map & (1 << curcpu)) != 0)
smp_rendezvous_action();
if (teardown_func == smp_no_rendevous_barrier)
while (atomic_load_acq_int(&smp_rv_waiters[2]) < ncpus)
cpu_spinwait();
/* release lock */
mtx_unlock_spin(&smp_ipi_mtx);
}
/*
* Handle an IPI sent to this processor.
*/
intrmask_t
smp_handle_ipi(struct trapframe *frame)
{
cpumask_t cpumask; /* This cpu mask */
u_int ipi, ipi_bitmap;
ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
cpumask = PCPU_GET(cpumask);
CTR1(KTR_SMP, "smp_handle_ipi(), ipi_bitmap=%x", ipi_bitmap);
while (ipi_bitmap) {
/*
* Find the lowest set bit.
*/
ipi = ipi_bitmap & ~(ipi_bitmap - 1);
ipi_bitmap &= ~ipi;
switch (ipi) {
case IPI_INVLTLB:
CTR0(KTR_SMP, "IPI_INVLTLB");
break;
case IPI_RENDEZVOUS:
CTR0(KTR_SMP, "IPI_RENDEZVOUS");
smp_rendezvous_action();
break;
case IPI_AST:
CTR0(KTR_SMP, "IPI_AST");
break;
case IPI_STOP:
/*
* IPI_STOP_HARD is mapped to IPI_STOP so it is not
* necessary to add it in the switch.
*/
CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD");
atomic_set_int(&stopped_cpus, cpumask);
while ((started_cpus & cpumask) == 0)
;
atomic_clear_int(&started_cpus, cpumask);
atomic_clear_int(&stopped_cpus, cpumask);
break;
}
}
return CR_INT_IPI;
}
static void
iv_rendezvous(uintptr_t a, uintptr_t b)
{
smp_rendezvous_action();
}
void
smp_rendezvous_cpus(cpuset_t map,
void (* setup_func)(void *),
void (* action_func)(void *),
void (* teardown_func)(void *),
void *arg)
{
int curcpumap, i, ncpus = 0;
/* Look comments in the !SMP case. */
if (!smp_started) {
spinlock_enter();
if (setup_func != NULL)
setup_func(arg);
if (action_func != NULL)
action_func(arg);
if (teardown_func != NULL)
teardown_func(arg);
spinlock_exit();
return;
}
CPU_FOREACH(i) {
if (CPU_ISSET(i, &map))
ncpus++;
}
if (ncpus == 0)
panic("ncpus is 0 with non-zero map");
mtx_lock_spin(&smp_ipi_mtx);
/* Pass rendezvous parameters via global variables. */
smp_rv_ncpus = ncpus;
smp_rv_setup_func = setup_func;
smp_rv_action_func = action_func;
smp_rv_teardown_func = teardown_func;
smp_rv_func_arg = arg;
smp_rv_waiters[1] = 0;
smp_rv_waiters[2] = 0;
smp_rv_waiters[3] = 0;
atomic_store_rel_int(&smp_rv_waiters[0], 0);
/*
* Signal other processors, which will enter the IPI with
* interrupts off.
*/
curcpumap = CPU_ISSET(curcpu, &map);
CPU_CLR(curcpu, &map);
ipi_selected(map, IPI_RENDEZVOUS);
/* Check if the current CPU is in the map */
if (curcpumap != 0)
smp_rendezvous_action();
/*
* Ensure that the master CPU waits for all the other
* CPUs to finish the rendezvous, so that smp_rv_*
* pseudo-structure and the arg are guaranteed to not
* be in use.
*/
while (atomic_load_acq_int(&smp_rv_waiters[3]) < ncpus)
cpu_spinwait();
mtx_unlock_spin(&smp_ipi_mtx);
}
void
interrupt(u_int64_t vector, struct trapframe *framep)
{
struct thread *td;
volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
td = curthread;
atomic_add_int(&td->td_intr_nesting_level, 1);
/*
* Handle ExtINT interrupts by generating an INTA cycle to
* read the vector.
*/
if (vector == 0) {
vector = ib->ib_inta;
printf("ExtINT interrupt: vector=%ld\n", vector);
}
if (vector == 255) {/* clock interrupt */
/* CTR0(KTR_INTR, "clock interrupt"); */
cnt.v_intr++;
#ifdef EVCNT_COUNTERS
clock_intr_evcnt.ev_count++;
#else
intrcnt[INTRCNT_CLOCK]++;
#endif
critical_enter();
#ifdef SMP
clks[PCPU_GET(cpuid)]++;
/* Only the BSP runs the real clock */
if (PCPU_GET(cpuid) == 0) {
#endif
handleclock(framep);
/* divide hz (1024) by 8 to get stathz (128) */
if ((++schedclk2 & 0x7) == 0)
statclock((struct clockframe *)framep);
#ifdef SMP
} else {
ia64_set_itm(ia64_get_itc() + itm_reload);
mtx_lock_spin(&sched_lock);
hardclock_process(curthread, TRAPF_USERMODE(framep));
if ((schedclk2 & 0x7) == 0)
statclock_process(curkse, TRAPF_PC(framep),
TRAPF_USERMODE(framep));
mtx_unlock_spin(&sched_lock);
}
#endif
critical_exit();
#ifdef SMP
} else if (vector == ipi_vector[IPI_AST]) {
asts[PCPU_GET(cpuid)]++;
CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
} else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
rdvs[PCPU_GET(cpuid)]++;
CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
smp_rendezvous_action();
} else if (vector == ipi_vector[IPI_STOP]) {
u_int32_t mybit = PCPU_GET(cpumask);
CTR1(KTR_SMP, "IPI_STOP, cpuid=%d", PCPU_GET(cpuid));
savectx(PCPU_GET(pcb));
stopped_cpus |= mybit;
while ((started_cpus & mybit) == 0)
/* spin */;
started_cpus &= ~mybit;
stopped_cpus &= ~mybit;
if (PCPU_GET(cpuid) == 0 && cpustop_restartfunc != NULL) {
void (*f)(void) = cpustop_restartfunc;
cpustop_restartfunc = NULL;
(*f)();
}
} else if (vector == ipi_vector[IPI_TEST]) {
CTR1(KTR_SMP, "IPI_TEST, cpuid=%d", PCPU_GET(cpuid));
mp_ipi_test++;
#endif
} else {
ints[PCPU_GET(cpuid)]++;
ia64_dispatch_intr(framep, vector);
}
atomic_subtract_int(&td->td_intr_nesting_level, 1);
}
static int
ipi_handler(void *arg)
{
u_int cpu, ipi;
cpu = PCPU_GET(cpuid);
ipi = pic_ipi_read((int)arg);
while ((ipi != 0x3ff)) {
switch (ipi) {
case IPI_RENDEZVOUS:
CTR0(KTR_SMP, "IPI_RENDEZVOUS");
smp_rendezvous_action();
break;
case IPI_AST:
CTR0(KTR_SMP, "IPI_AST");
break;
case IPI_STOP:
/*
* IPI_STOP_HARD is mapped to IPI_STOP so it is not
* necessary to add it in the switch.
*/
CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD");
savectx(&stoppcbs[cpu]);
/*
* CPUs are stopped when entering the debugger and at
* system shutdown, both events which can precede a
* panic dump. For the dump to be correct, all caches
* must be flushed and invalidated, but on ARM there's
* no way to broadcast a wbinv_all to other cores.
* Instead, we have each core do the local wbinv_all as
* part of stopping the core. The core requesting the
* stop will do the l2 cache flush after all other cores
* have done their l1 flushes and stopped.
*/
cpu_idcache_wbinv_all();
/* Indicate we are stopped */
CPU_SET_ATOMIC(cpu, &stopped_cpus);
/* Wait for restart */
while (!CPU_ISSET(cpu, &started_cpus))
cpu_spinwait();
CPU_CLR_ATOMIC(cpu, &started_cpus);
CPU_CLR_ATOMIC(cpu, &stopped_cpus);
CTR0(KTR_SMP, "IPI_STOP (restart)");
break;
case IPI_PREEMPT:
CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
sched_preempt(curthread);
break;
case IPI_HARDCLOCK:
CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
hardclockintr();
break;
case IPI_TLB:
CTR1(KTR_SMP, "%s: IPI_TLB", __func__);
cpufuncs.cf_tlb_flushID();
break;
default:
panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu);
}
pic_ipi_clear(ipi);
ipi = pic_ipi_read(-1);
}
return (FILTER_HANDLED);
}
