void smp_rendezvous_cpus(unsigned long map, void (* action_func)(void *), void *arg) { unsigned int cpumask = 1 << cpu_number(); if (ncpus == 1) { if (action_func != NULL) action_func(arg); return; } /* obtain rendezvous lock */ mtx_enter(&smp_ipi_mtx); /* set static function pointers */ smp_rv_map = map; smp_rv_action_func = action_func; smp_rv_func_arg = arg; smp_rv_waiters[0] = 0; smp_rv_waiters[1] = 0; /* signal other processors, which will enter the IPI with interrupts off */ mips64_multicast_ipi(map & ~cpumask, MIPS64_IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ if (map & cpumask) smp_rendezvous_action(); while (smp_rv_waiters[1] != smp_rv_map) ; /* release lock */ mtx_leave(&smp_ipi_mtx); }
void smp_rendezvous_cpus(cpumask_t map, void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { int i, ncpus = 0; if (!smp_started) { if (setup_func != NULL) setup_func(arg); if (action_func != NULL) action_func(arg); if (teardown_func != NULL) teardown_func(arg); return; } CPU_FOREACH(i) { if (((1 << i) & map) != 0) ncpus++; } if (ncpus == 0) panic("ncpus is 0 with map=0x%x", map); /* obtain rendezvous lock */ mtx_lock_spin(&smp_ipi_mtx); /* set static function pointers */ smp_rv_ncpus = ncpus; smp_rv_setup_func = setup_func; smp_rv_action_func = action_func; smp_rv_teardown_func = teardown_func; smp_rv_func_arg = arg; smp_rv_waiters[1] = 0; smp_rv_waiters[2] = 0; atomic_store_rel_int(&smp_rv_waiters[0], 0); /* signal other processors, which will enter the IPI with interrupts off */ ipi_selected(map & ~(1 << curcpu), IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ if ((map & (1 << curcpu)) != 0) smp_rendezvous_action(); if (teardown_func == smp_no_rendevous_barrier) while (atomic_load_acq_int(&smp_rv_waiters[2]) < ncpus) cpu_spinwait(); /* release lock */ mtx_unlock_spin(&smp_ipi_mtx); }
/* * Handle an IPI sent to this processor. */ intrmask_t smp_handle_ipi(struct trapframe *frame) { cpumask_t cpumask; /* This cpu mask */ u_int ipi, ipi_bitmap; ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); cpumask = PCPU_GET(cpumask); CTR1(KTR_SMP, "smp_handle_ipi(), ipi_bitmap=%x", ipi_bitmap); while (ipi_bitmap) { /* * Find the lowest set bit. */ ipi = ipi_bitmap & ~(ipi_bitmap - 1); ipi_bitmap &= ~ipi; switch (ipi) { case IPI_INVLTLB: CTR0(KTR_SMP, "IPI_INVLTLB"); break; case IPI_RENDEZVOUS: CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); break; case IPI_AST: CTR0(KTR_SMP, "IPI_AST"); break; case IPI_STOP: /* * IPI_STOP_HARD is mapped to IPI_STOP so it is not * necessary to add it in the switch. */ CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); atomic_set_int(&stopped_cpus, cpumask); while ((started_cpus & cpumask) == 0) ; atomic_clear_int(&started_cpus, cpumask); atomic_clear_int(&stopped_cpus, cpumask); break; } } return CR_INT_IPI; }
static void iv_rendezvous(uintptr_t a, uintptr_t b) { smp_rendezvous_action(); }
void smp_rendezvous_cpus(cpuset_t map, void (* setup_func)(void *), void (* action_func)(void *), void (* teardown_func)(void *), void *arg) { int curcpumap, i, ncpus = 0; /* Look comments in the !SMP case. */ if (!smp_started) { spinlock_enter(); if (setup_func != NULL) setup_func(arg); if (action_func != NULL) action_func(arg); if (teardown_func != NULL) teardown_func(arg); spinlock_exit(); return; } CPU_FOREACH(i) { if (CPU_ISSET(i, &map)) ncpus++; } if (ncpus == 0) panic("ncpus is 0 with non-zero map"); mtx_lock_spin(&smp_ipi_mtx); /* Pass rendezvous parameters via global variables. */ smp_rv_ncpus = ncpus; smp_rv_setup_func = setup_func; smp_rv_action_func = action_func; smp_rv_teardown_func = teardown_func; smp_rv_func_arg = arg; smp_rv_waiters[1] = 0; smp_rv_waiters[2] = 0; smp_rv_waiters[3] = 0; atomic_store_rel_int(&smp_rv_waiters[0], 0); /* * Signal other processors, which will enter the IPI with * interrupts off. */ curcpumap = CPU_ISSET(curcpu, &map); CPU_CLR(curcpu, &map); ipi_selected(map, IPI_RENDEZVOUS); /* Check if the current CPU is in the map */ if (curcpumap != 0) smp_rendezvous_action(); /* * Ensure that the master CPU waits for all the other * CPUs to finish the rendezvous, so that smp_rv_* * pseudo-structure and the arg are guaranteed to not * be in use. */ while (atomic_load_acq_int(&smp_rv_waiters[3]) < ncpus) cpu_spinwait(); mtx_unlock_spin(&smp_ipi_mtx); }
void interrupt(u_int64_t vector, struct trapframe *framep) { struct thread *td; volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK; td = curthread; atomic_add_int(&td->td_intr_nesting_level, 1); /* * Handle ExtINT interrupts by generating an INTA cycle to * read the vector. */ if (vector == 0) { vector = ib->ib_inta; printf("ExtINT interrupt: vector=%ld\n", vector); } if (vector == 255) {/* clock interrupt */ /* CTR0(KTR_INTR, "clock interrupt"); */ cnt.v_intr++; #ifdef EVCNT_COUNTERS clock_intr_evcnt.ev_count++; #else intrcnt[INTRCNT_CLOCK]++; #endif critical_enter(); #ifdef SMP clks[PCPU_GET(cpuid)]++; /* Only the BSP runs the real clock */ if (PCPU_GET(cpuid) == 0) { #endif handleclock(framep); /* divide hz (1024) by 8 to get stathz (128) */ if ((++schedclk2 & 0x7) == 0) statclock((struct clockframe *)framep); #ifdef SMP } else { ia64_set_itm(ia64_get_itc() + itm_reload); mtx_lock_spin(&sched_lock); hardclock_process(curthread, TRAPF_USERMODE(framep)); if ((schedclk2 & 0x7) == 0) statclock_process(curkse, TRAPF_PC(framep), TRAPF_USERMODE(framep)); mtx_unlock_spin(&sched_lock); } #endif critical_exit(); #ifdef SMP } else if (vector == ipi_vector[IPI_AST]) { asts[PCPU_GET(cpuid)]++; CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); } else if (vector == ipi_vector[IPI_RENDEZVOUS]) { rdvs[PCPU_GET(cpuid)]++; CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); smp_rendezvous_action(); } else if (vector == ipi_vector[IPI_STOP]) { u_int32_t mybit = PCPU_GET(cpumask); CTR1(KTR_SMP, "IPI_STOP, cpuid=%d", PCPU_GET(cpuid)); savectx(PCPU_GET(pcb)); stopped_cpus |= mybit; while ((started_cpus & mybit) == 0) /* spin */; started_cpus &= ~mybit; stopped_cpus &= ~mybit; if (PCPU_GET(cpuid) == 0 && cpustop_restartfunc != NULL) { void (*f)(void) = cpustop_restartfunc; cpustop_restartfunc = NULL; (*f)(); } } else if (vector == ipi_vector[IPI_TEST]) { CTR1(KTR_SMP, "IPI_TEST, cpuid=%d", PCPU_GET(cpuid)); mp_ipi_test++; #endif } else { ints[PCPU_GET(cpuid)]++; ia64_dispatch_intr(framep, vector); } atomic_subtract_int(&td->td_intr_nesting_level, 1); }
static int ipi_handler(void *arg) { u_int cpu, ipi; cpu = PCPU_GET(cpuid); ipi = pic_ipi_read((int)arg); while ((ipi != 0x3ff)) { switch (ipi) { case IPI_RENDEZVOUS: CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); break; case IPI_AST: CTR0(KTR_SMP, "IPI_AST"); break; case IPI_STOP: /* * IPI_STOP_HARD is mapped to IPI_STOP so it is not * necessary to add it in the switch. */ CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); savectx(&stoppcbs[cpu]); /* * CPUs are stopped when entering the debugger and at * system shutdown, both events which can precede a * panic dump. For the dump to be correct, all caches * must be flushed and invalidated, but on ARM there's * no way to broadcast a wbinv_all to other cores. * Instead, we have each core do the local wbinv_all as * part of stopping the core. The core requesting the * stop will do the l2 cache flush after all other cores * have done their l1 flushes and stopped. */ cpu_idcache_wbinv_all(); /* Indicate we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) cpu_spinwait(); CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); CTR0(KTR_SMP, "IPI_STOP (restart)"); break; case IPI_PREEMPT: CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); sched_preempt(curthread); break; case IPI_HARDCLOCK: CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); hardclockintr(); break; case IPI_TLB: CTR1(KTR_SMP, "%s: IPI_TLB", __func__); cpufuncs.cf_tlb_flushID(); break; default: panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu); } pic_ipi_clear(ipi); ipi = pic_ipi_read(-1); } return (FILTER_HANDLED); }