/* * callout_hardclock: * * Called from hardclock() once every tick. We schedule a soft * interrupt if there is work to be done. */ void callout_hardclock(void) { struct callout_cpu *cc; int needsoftclock, ticks; cc = curcpu()->ci_data.cpu_callout; mutex_spin_enter(cc->cc_lock); ticks = ++cc->cc_ticks; MOVEBUCKET(cc, 0, ticks); if (MASKWHEEL(0, ticks) == 0) { MOVEBUCKET(cc, 1, ticks); if (MASKWHEEL(1, ticks) == 0) { MOVEBUCKET(cc, 2, ticks); if (MASKWHEEL(2, ticks) == 0) MOVEBUCKET(cc, 3, ticks); } } needsoftclock = !CIRCQ_EMPTY(&cc->cc_todo); mutex_spin_exit(cc->cc_lock); if (needsoftclock) softint_schedule(callout_sih); }
static inline void xpq_increment_idx(void) { if (__predict_false(++xpq_idx_array[curcpu()->ci_cpuid] == XPQUEUE_SIZE)) xpq_flush_queue(); }
void arm_dflt_setipl(int newcpl) { struct cpu_info *ci = curcpu(); ci->ci_cpl = newcpl; }
void cpu_set_curpri(int pri) { kpreempt_disable(); curcpu()->ci_schedstate.spc_curpriority = pri; kpreempt_enable(); }
void dosoftint() { struct cpu_info *ci = curcpu(); int sir, q, mask; #ifdef MULTIPROCESSOR register_t sr; /* Enable interrupts */ sr = getsr(); ENABLEIPI(); __mp_lock(&kernel_lock); #endif while ((sir = ci->ci_softpending) != 0) { atomic_clearbits_int(&ci->ci_softpending, sir); for (q = SI_NQUEUES - 1; q >= 0; q--) { mask = SINTMASK(q); if (sir & mask) softintr_dispatch(q); } } #ifdef MULTIPROCESSOR __mp_unlock(&kernel_lock); setsr(sr); #endif }
void db_cpu_cmd(db_expr_t addr, bool have_addr, db_expr_t count, const char *modif) { struct cpu_info *ci; if (!have_addr) { cpu_debug_dump(); return; } if ((addr < 0) || (addr >= sparc_ncpus)) { db_printf("%ld: CPU out of range\n", addr); return; } ci = cpus[addr]; if (ci == NULL) { db_printf("CPU %ld not configured\n", addr); return; } if (ci != curcpu()) { if (!(ci->flags & CPUFLG_PAUSED)) { db_printf("CPU %ld not paused\n", addr); return; } } if (ci->ci_ddb_regs == 0) { db_printf("CPU %ld has no saved regs\n", addr); return; } db_printf("using CPU %ld", addr); ddb_regp = __UNVOLATILE(ci->ci_ddb_regs); ddb_cpuinfo = ci; }
static void tegra_cpufreq_post(void *arg1, void *arg2) { struct cpu_info *ci = curcpu(); ci->ci_data.cpu_cc_freq = cpufreq_get_rate() * 1000000; }
int procfs_douptime(struct lwp *curl, struct proc *p, struct pfsnode *pfs, struct uio *uio) { char *bf; int len; struct timeval runtime; u_int64_t idle; int error = 0; bf = malloc(LBFSZ, M_TEMP, M_WAITOK); microuptime(&runtime); idle = curcpu()->ci_schedstate.spc_cp_time[CP_IDLE]; len = snprintf(bf, LBFSZ, "%lu.%02lu %" PRIu64 ".%02" PRIu64 "\n", runtime.tv_sec, runtime.tv_usec / 10000, idle / hz, (((idle % hz) * 100) / hz) % 100); if (len == 0) goto out; error = uiomove_frombuf(bf, len, uio); out: free(bf, M_TEMP); return error; }
/* * Called with interrupts enabled. */ void acpicpu_md_cstate_enter(int method, int state) { struct cpu_info *ci = curcpu(); KASSERT(ci->ci_ilevel == IPL_NONE); switch (method) { case ACPICPU_C_STATE_FFH: x86_monitor(&ci->ci_want_resched, 0, 0); if (__predict_false(ci->ci_want_resched != 0)) return; x86_mwait((state - 1) << 4, 0); break; case ACPICPU_C_STATE_HALT: x86_disable_intr(); if (__predict_false(ci->ci_want_resched != 0)) { x86_enable_intr(); return; } x86_stihlt(); break; } }
static __inline void userret (struct lwp *l, register_t pc, u_quad_t oticks) { struct proc *p = l->l_proc; int sig; /* take pending signals */ while ((sig = CURSIG(l)) != 0) postsig(sig); l->l_priority = l->l_usrpri; if (want_resched) { /* * We're being preempted. */ preempt(0); while ((sig = CURSIG(l)) != 0) postsig(sig); } /* * If profiling, charge recent system time to the trapped pc. */ if (l->l_flag & P_PROFIL) { extern int psratio; addupc_task(p, pc, (int)(p->p_sticks - oticks) * psratio); } curcpu()->ci_schedstate.spc_curpriority = l->l_priority; }
int mtx_enter_try(struct mutex *mtx) { struct cpu_info *owner, *ci = curcpu(); int s; if (mtx->mtx_wantipl != IPL_NONE) s = splraise(mtx->mtx_wantipl); owner = atomic_cas_ptr(&mtx->mtx_owner, NULL, ci); #ifdef DIAGNOSTIC if (__predict_false(owner == ci)) panic("mtx %p: locking against myself", mtx); #endif if (owner == NULL) { if (mtx->mtx_wantipl != IPL_NONE) mtx->mtx_oldipl = s; #ifdef DIAGNOSTIC ci->ci_mutex_level++; #endif membar_enter(); return (1); } if (mtx->mtx_wantipl != IPL_NONE) splx(s); return (0); }
void * percpu_getref(percpu_t *pc) { kpreempt_disable(); return percpu_getptr_remote(pc, curcpu()); }
/* * called once per clock tick via the pps callback * for the calibration of the TSC counters. * it is called only for the PRIMARY cpu. all * other cpus are called via a broadcast IPI * calibration interval is 1 second - we call * the calibration code only every hz calls */ static void cc_calibrate(struct timecounter *tc) { static int calls; struct cpu_info *ci; KASSERT(kpreempt_disabled()); /* * XXX: for high interrupt frequency * support: ++calls < hz / tc_tick */ if (++calls < hz) return; calls = 0; ci = curcpu(); /* pick up reference ticks */ cc_cal_val = cpu_counter32(); #if defined(MULTIPROCESSOR) cc_calibrate_mp(ci); #endif cc_calibrate_cpu(ci); }
/* * Further secondary CPU initialization. * * We are now running on our startup stack, with proper page tables. * There is nothing to do but display some details about the CPU and its CMMUs. */ void secondary_main() { struct cpu_info *ci = curcpu(); int s; cpu_configuration_print(0); ncpus++; sched_init_cpu(ci); nanouptime(&ci->ci_schedstate.spc_runtime); ci->ci_curproc = NULL; ci->ci_randseed = (arc4random() & 0x7fffffff) + 1; /* * Release cpu_hatch_mutex to let other secondary processors * have a chance to run. */ hatch_pending_count--; __cpu_simple_unlock(&cpu_hatch_mutex); /* wait for cpu_boot_secondary_processors() */ __cpu_simple_lock(&cpu_boot_mutex); __cpu_simple_unlock(&cpu_boot_mutex); spl0(); SCHED_LOCK(s); set_psr(get_psr() & ~PSR_IND); SET(ci->ci_flags, CIF_ALIVE); cpu_switchto(NULL, sched_chooseproc()); }
/* * Same as above, but also handles writeback completion on 68040. */ void wb_userret(struct proc *p, struct frame *fp) { int sig; union sigval sv; /* take pending signals */ while ((sig = CURSIG(p)) != 0) postsig(sig); p->p_priority = p->p_usrpri; /* * Deal with user mode writebacks (from trap, or from sigreturn). * If any writeback fails, go back and attempt signal delivery * unless we have already been here and attempted the writeback * (e.g. bad address with user ignoring SIGSEGV). In that case, * we just return to the user without successfully completing * the writebacks. Maybe we should just drop the sucker? */ if (mmutype == MMU_68040 && fp->f_format == FMT7) { if ((sig = writeback(fp)) != 0) { sv.sival_ptr = (void *)fp->f_fmt7.f_fa; trapsignal(p, sig, T_MMUFLT, SEGV_MAPERR, sv); while ((sig = CURSIG(p)) != 0) postsig(sig); p->p_priority = p->p_usrpri; } } curcpu()->ci_schedstate.spc_curpriority = p->p_priority; }
void cpu_intr(int ppl, vaddr_t pc, uint32_t status) { struct cpu_info * const ci = curcpu(); uint32_t pending; int ipl; #ifdef DIAGNOSTIC const int mtx_count = ci->ci_mtx_count; const u_int biglock_count = ci->ci_biglock_count; const u_int blcnt = curlwp->l_blcnt; #endif KASSERT(ci->ci_cpl == IPL_HIGH); KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); ci->ci_data.cpu_nintr++; while (ppl < (ipl = splintr(&pending))) { KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); splx(ipl); /* lower to interrupt level */ KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); KASSERTMSG(ci->ci_cpl == ipl, "%s: cpl (%d) != ipl (%d)", __func__, ci->ci_cpl, ipl); KASSERT(pending != 0); cf.pc = pc; cf.sr = status; cf.intr = (ci->ci_idepth > 1); #ifdef MIPS3_ENABLE_CLOCK_INTR if (pending & MIPS_INT_MASK_5) { KASSERTMSG(ipl == IPL_SCHED, "%s: ipl (%d) != IPL_SCHED (%d)", __func__, ipl, IPL_SCHED); /* call the common MIPS3 clock interrupt handler */ mips3_clockintr(&cf); pending ^= MIPS_INT_MASK_5; } #endif if (pending != 0) { /* Process I/O and error interrupts. */ evbmips_iointr(ipl, pc, pending); } KASSERT(biglock_count == ci->ci_biglock_count); KASSERT(blcnt == curlwp->l_blcnt); KASSERT(mtx_count == ci->ci_mtx_count); /* * If even our spl is higher now (due to interrupting while * spin-lock is held and higher IPL spin-lock is locked, it * can no longer be locked so it's safe to lower IPL back * to ppl. */ (void) splhigh(); /* disable interrupts */ } KASSERT(ci->ci_cpl == IPL_HIGH); KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); }
static void amlogic_cpufreq_cb(void *arg1, void *arg2) { struct cpu_info *ci = curcpu(); ci->ci_data.cpu_cc_freq = cpufreq_get_rate() * 1000000; }
void cpu_idle_mwait_cycle(void) { struct cpu_info *ci = curcpu(); if ((read_rflags() & PSL_I) == 0) panic("idle with interrupts blocked!"); /* something already queued? */ if (!cpu_is_idle(ci)) return; /* * About to idle; setting the MWAIT_IN_IDLE bit tells * cpu_unidle() that it can't be a no-op and tells cpu_kick() * that it doesn't need to use an IPI. We also set the * MWAIT_KEEP_IDLING bit: those routines clear it to stop * the mwait. Once they're set, we do a final check of the * queue, in case another cpu called setrunqueue() and added * something to the queue and called cpu_unidle() between * the check in sched_idle() and here. */ atomic_setbits_int(&ci->ci_mwait, MWAIT_IDLING | MWAIT_ONLY); if (cpu_is_idle(ci)) { monitor(&ci->ci_mwait, 0, 0); if ((ci->ci_mwait & MWAIT_IDLING) == MWAIT_IDLING) mwait(0, 0); } /* done idling; let cpu_kick() know that an IPI is required */ atomic_clearbits_int(&ci->ci_mwait, MWAIT_IDLING); }
void __mp_lock(struct __mp_lock *mpl) { int s; struct cpu_info *ci = curcpu(); /* * Please notice that mpl_count gets incremented twice for the * first lock. This is on purpose. The way we release the lock * in mp_unlock is to decrement the mpl_count and then check if * the lock should be released. Since mpl_count is what we're * spinning on, decrementing it in mpl_unlock to 0 means that * we can't clear mpl_cpu, because we're no longer holding the * lock. In theory mpl_cpu doesn't need to be cleared, but it's * safer to clear it and besides, setting mpl_count to 2 on the * first lock makes most of this code much simpler. */ while (1) { s = splhigh(); if (__cpu_cas(&mpl->mpl_count, 0, 1) == 0) { alpha_mb(); mpl->mpl_cpu = ci; } if (mpl->mpl_cpu == ci) { mpl->mpl_count++; splx(s); break; } splx(s); __mp_lock_spin(mpl); } }
static void cal_timer(void) { uint32_t cntfreq; cntfreq = curcpu()->ci_cpu_freq = RA_CLOCK_RATE; /* MIPS 4Kc CP0 counts every other clock */ if (mips_options.mips_cpu_flags & CPU_MIPS_DOUBLE_COUNT) cntfreq /= 2; curcpu()->ci_cycles_per_hz = (cntfreq + hz / 2) / hz; /* Compute number of cycles per 1us (1/MHz). 0.5MHz is for roundup. */ curcpu()->ci_divisor_delay = ((cntfreq + 500000) / 1000000); }
/* * cpuwatch_set - establish a MIPS COP0 watchpoint */ void cpuwatch_set(cpu_watchpoint_t *cwp) { struct cpu_info * const ci = curcpu(); uint32_t watchhi; register_t watchlo; int cwnum = cwp - &ci->ci_cpuwatch_tab[0]; KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); watchlo = cwp->cw_addr; if (cwp->cw_mode & CPUWATCH_WRITE) watchlo |= __BIT(0); if (cwp->cw_mode & CPUWATCH_READ) watchlo |= __BIT(1); if (cwp->cw_mode & CPUWATCH_EXEC) watchlo |= __BIT(2); if (cwp->cw_mode & CPUWATCH_ASID) watchhi = cwp->cw_asid << 16; /* addr qualified by asid */ else watchhi = __BIT(30); /* addr not qual. by asid (Global) */ if (cwp->cw_mode & CPUWATCH_MASK) watchhi |= cwp->cw_mask; /* set "dont care" addr match bits */ mipsNN_cp0_watchhi_write(cwnum, watchhi); mipsNN_cp0_watchlo_write(cwnum, watchlo); }
/* * Maskable IPIs. * * These IPIs are received as non maskable, but are not processed in * the NMI handler; instead, they are processed from the soft interrupt * handler. * * XXX This is grossly suboptimal. */ void m197_soft_ipi() { struct cpu_info *ci = curcpu(); struct trapframe faketf; int s; __mp_lock(&kernel_lock); s = splclock(); if (ci->ci_h_sxip != 0) { faketf.tf_cpu = ci; faketf.tf_sxip = ci->ci_h_sxip; faketf.tf_epsr = ci->ci_h_epsr; ci->ci_h_sxip = 0; hardclock((struct clockframe *)&faketf); } if (ci->ci_s_sxip != 0) { faketf.tf_cpu = ci; faketf.tf_sxip = ci->ci_s_sxip; faketf.tf_epsr = ci->ci_s_epsr; ci->ci_s_sxip = 0; statclock((struct clockframe *)&faketf); } splx(s); __mp_unlock(&kernel_lock); }
/* * Pause this cpu */ void cpu_pause(struct reg *regsp) { int s = splhigh(); cpuid_t cii = cpu_index(curcpu()); if (__predict_false(cold)) return; do { kcpuset_atomic_set(cpus_paused, cii); do { ; } while (kcpuset_isset(cpus_paused, cii)); kcpuset_atomic_set(cpus_resumed, cii); #if defined(DDB) if (ddb_running_on_this_cpu_p()) cpu_Debugger(); if (ddb_running_on_any_cpu_p()) continue; #endif } while (false); splx(s); }
void * percpu_getref(percpu_t *pc) { KPREEMPT_DISABLE(curlwp); return percpu_getptr_remote(pc, curcpu()); }
/* * Independent profiling "tick" in case we're using a separate * clock or profiling event source. Currently, that's just * performance counters--hence the wrapper. */ void proftick(struct clockframe *frame) { #ifdef GPROF struct gmonparam *g; intptr_t i; #endif struct lwp *l; struct proc *p; l = curcpu()->ci_data.cpu_onproc; p = (l ? l->l_proc : NULL); if (CLKF_USERMODE(frame)) { mutex_spin_enter(&p->p_stmutex); if (p->p_stflag & PST_PROFIL) addupc_intr(l, CLKF_PC(frame)); mutex_spin_exit(&p->p_stmutex); } else { #ifdef GPROF g = &_gmonparam; if (g->state == GMON_PROF_ON) { i = CLKF_PC(frame) - g->lowpc; if (i < g->textsize) { i /= HISTFRACTION * sizeof(*g->kcount); g->kcount[i]++; } } #endif #ifdef LWP_PC if (p != NULL && (p->p_stflag & PST_PROFIL) != 0) addupc_intr(l, LWP_PC(l)); #endif } }
void userret(struct lwp *l) { #if defined(__PROG32) && defined(ARM_MMU_EXTENDED) /* * If our ASID got released, access via TTBR0 will have been disabled. * So if it is disabled, activate the lwp again to get a new ASID. */ #ifdef __HAVE_PREEMPTION kpreempt_disable(); #endif KASSERT(curcpu()->ci_pmap_cur == l->l_proc->p_vmspace->vm_map.pmap); if (__predict_false(armreg_ttbcr_read() & TTBCR_S_PD0)) { pmap_activate(l); } KASSERT(!(armreg_ttbcr_read() & TTBCR_S_PD0)); #ifdef __HAVE_PREEMPTION kpreempt_enable(); #endif #endif /* Invoke MI userret code */ mi_userret(l); #if defined(__PROG32) && defined(DIAGNOSTIC) KASSERT(VALID_R15_PSR(lwp_trapframe(l)->tf_pc, lwp_trapframe(l)->tf_spsr)); #endif }
static void openpic_send_ipi(cpuid_t target, uint32_t mesg) { struct cpu_info * const ci = curcpu(); uint32_t cpumask = 0; switch (target) { case IPI_DST_ALL: case IPI_DST_NOTME: for (u_int i = 0; i < ncpu; i++) { struct cpu_info * const dst_ci = cpu_lookup(i); if (target == IPI_DST_ALL || dst_ci != ci) { cpumask |= 1 << cpu_index(dst_ci); atomic_or_32(&dst_ci->ci_pending_ipis, mesg); } } break; default: { struct cpu_info * const dst_ci = cpu_lookup(target); cpumask = 1 << cpu_index(dst_ci); atomic_or_32(&dst_ci->ci_pending_ipis, mesg); break; } } openpic_write(OPENPIC_IPI(cpu_index(ci), 1), cpumask); }
void ampscu_attach(struct device *parent, struct device *self, void *args) { struct ampscu_softc *sc = (struct ampscu_softc *)self; struct cortex_attach_args *ca = args; sc->sc_iot = ca->ca_iot; if (bus_space_map(sc->sc_iot, ca->ca_periphbase + SCU_ADDR, SCU_SIZE, 0, &sc->sc_ioh)) panic("ampscu_attach: bus_space_map failed!"); ncpusfound = ampscu_ncpus(sc); printf(": %d CPUs\n", ncpusfound); #ifdef MULTIPROCESSOR /* ARM Errata 764369 */ if ((curcpu()->ci_arm_cpuid & CPU_ID_CORTEX_A9_MASK) == CPU_ID_CORTEX_A9) bus_space_write_4(sc->sc_iot, sc->sc_ioh, 0x30, bus_space_read_4(sc->sc_iot, sc->sc_ioh, 0x30) | 1); bus_space_write_4(sc->sc_iot, sc->sc_ioh, SCU_CTRL, bus_space_read_4(sc->sc_iot, sc->sc_ioh, SCU_CTRL) | 1); /* Flush ALL the caches. */ cpu_drain_writebuf(); cpu_idcache_wbinv_all(); cpu_sdcache_wbinv_all(); cpu_drain_writebuf(); #endif }
/* * uvm_emap_switch: if the CPU is 'behind' the LWP in emap visibility, * perform TLB flush and thus update the local view. Main purpose is * to handle kernel preemption, while emap is in use. * * => called from mi_switch(), when LWP returns after block or preempt. */ void uvm_emap_switch(lwp_t *l) { struct uvm_cpu *ucpu; u_int curgen, gen; KASSERT(kpreempt_disabled()); /* If LWP did not use emap, then nothing to do. */ if (__predict_true(l->l_emap_gen == UVM_EMAP_INACTIVE)) { return; } /* * No need to synchronise if generation number of current CPU is * newer than the number of this LWP. * * This test assumes two's complement arithmetic and allows * ~2B missed updates before it will produce bad results. */ ucpu = curcpu()->ci_data.cpu_uvm; curgen = ucpu->emap_gen; gen = l->l_emap_gen; if (__predict_true((signed int)(curgen - gen) >= 0)) { return; } /* * See comments in uvm_emap_consume() about memory * barriers and race conditions. */ curgen = uvm_emap_gen_return(); pmap_emap_sync(false); ucpu->emap_gen = curgen; }
/* * Send an interprocessor interrupt - sun4u. */ void sparc64_send_ipi_sun4u(int upaid, ipifunc_t func, uint64_t arg1, uint64_t arg2) { int i, ik, shift = 0; uint64_t intr_func; KASSERT(upaid != curcpu()->ci_cpuid); /* * UltraSPARC-IIIi CPUs select the BUSY/NACK pair based on the * lower two bits of the ITID. */ if (CPU_IS_USIIIi()) shift = (upaid & 0x3) * 2; if (ldxa(0, ASI_IDSR) & (IDSR_BUSY << shift)) panic("recursive IPI?"); intr_func = (uint64_t)(u_long)func; /* Schedule an interrupt. */ for (i = 0; i < 10000; i++) { int s = intr_disable(); stxa(IDDR_0H, ASI_INTERRUPT_DISPATCH, intr_func); stxa(IDDR_1H, ASI_INTERRUPT_DISPATCH, arg1); stxa(IDDR_2H, ASI_INTERRUPT_DISPATCH, arg2); stxa(IDCR(upaid), ASI_INTERRUPT_DISPATCH, 0); membar_Sync(); /* Workaround for SpitFire erratum #54, from FreeBSD */ if (CPU_IS_SPITFIRE()) { (void)ldxa(P_DCR_0, ASI_INTERRUPT_RECEIVE_DATA); membar_Sync(); } for (ik = 0; ik < 1000000; ik++) { if (ldxa(0, ASI_IDSR) & (IDSR_BUSY << shift)) continue; else break; } intr_restore(s); if (ik == 1000000) break; if ((ldxa(0, ASI_IDSR) & (IDSR_NACK << shift)) == 0) return; /* * Wait for a while with enabling interrupts to avoid * deadlocks. XXX - random value is better. */ DELAY(1); } if (panicstr == NULL) panic("cpu%d: ipi_send: couldn't send ipi to UPAID %u" " (tried %d times)", cpu_number(), upaid, i); }