/*
 * callout_hardclock:
 *
 *	Called from hardclock() once every tick.  We schedule a soft
 *	interrupt if there is work to be done.
 */
void
callout_hardclock(void)
{
	struct callout_cpu *cc;
	int needsoftclock, ticks;

	cc = curcpu()->ci_data.cpu_callout;
	mutex_spin_enter(cc->cc_lock);

	ticks = ++cc->cc_ticks;

	MOVEBUCKET(cc, 0, ticks);
	if (MASKWHEEL(0, ticks) == 0) {
		MOVEBUCKET(cc, 1, ticks);
		if (MASKWHEEL(1, ticks) == 0) {
			MOVEBUCKET(cc, 2, ticks);
			if (MASKWHEEL(2, ticks) == 0)
				MOVEBUCKET(cc, 3, ticks);
		}
	}

	needsoftclock = !CIRCQ_EMPTY(&cc->cc_todo);
	mutex_spin_exit(cc->cc_lock);

	if (needsoftclock)
		softint_schedule(callout_sih);
}
Example #2
0
static inline void
xpq_increment_idx(void)
{

	if (__predict_false(++xpq_idx_array[curcpu()->ci_cpuid] == XPQUEUE_SIZE))
		xpq_flush_queue();
}
Example #3
0
void
arm_dflt_setipl(int newcpl)
{
	struct cpu_info *ci = curcpu();

	ci->ci_cpl = newcpl;
}
Example #4
0
void
cpu_set_curpri(int pri)
{
	kpreempt_disable();
	curcpu()->ci_schedstate.spc_curpriority = pri;
	kpreempt_enable();
}
Example #5
0
void
dosoftint()
{
	struct cpu_info *ci = curcpu();
	int sir, q, mask;
#ifdef MULTIPROCESSOR
	register_t sr;

	/* Enable interrupts */
	sr = getsr();
	ENABLEIPI();
	__mp_lock(&kernel_lock);
#endif

	while ((sir = ci->ci_softpending) != 0) {
		atomic_clearbits_int(&ci->ci_softpending, sir);

		for (q = SI_NQUEUES - 1; q >= 0; q--) {
			mask = SINTMASK(q);
			if (sir & mask)
				softintr_dispatch(q);
		}
	}

#ifdef MULTIPROCESSOR
	__mp_unlock(&kernel_lock);
	setsr(sr);
#endif
}
void
db_cpu_cmd(db_expr_t addr, bool have_addr, db_expr_t count, const char *modif)
{
	struct cpu_info *ci;
	if (!have_addr) {
		cpu_debug_dump();
		return;
	}

	if ((addr < 0) || (addr >= sparc_ncpus)) {
		db_printf("%ld: CPU out of range\n", addr);
		return;
	}
	ci = cpus[addr];
	if (ci == NULL) {
		db_printf("CPU %ld not configured\n", addr);
		return;
	}
	if (ci != curcpu()) {
		if (!(ci->flags & CPUFLG_PAUSED)) {
			db_printf("CPU %ld not paused\n", addr);
			return;
		}
	}
	if (ci->ci_ddb_regs == 0) {
		db_printf("CPU %ld has no saved regs\n", addr);
		return;
	}
	db_printf("using CPU %ld", addr);
	ddb_regp = __UNVOLATILE(ci->ci_ddb_regs);
	ddb_cpuinfo = ci;
}
Example #7
0
static void
tegra_cpufreq_post(void *arg1, void *arg2)
{
	struct cpu_info *ci = curcpu();

	ci->ci_data.cpu_cc_freq = cpufreq_get_rate() * 1000000;
}
Example #8
0
int
procfs_douptime(struct lwp *curl, struct proc *p,
    struct pfsnode *pfs, struct uio *uio)
{
	char *bf;
	int len;
	struct timeval runtime;
	u_int64_t idle;
	int error = 0;

	bf = malloc(LBFSZ, M_TEMP, M_WAITOK);

	microuptime(&runtime);
	idle = curcpu()->ci_schedstate.spc_cp_time[CP_IDLE];
	len = snprintf(bf, LBFSZ,
	    "%lu.%02lu %" PRIu64 ".%02" PRIu64 "\n",
	    runtime.tv_sec, runtime.tv_usec / 10000,
	    idle / hz, (((idle % hz) * 100) / hz) % 100);

	if (len == 0)
		goto out;

	error = uiomove_frombuf(bf, len, uio);
out:
	free(bf, M_TEMP);
	return error;
}
Example #9
0
/*
 * Called with interrupts enabled.
 */
void
acpicpu_md_cstate_enter(int method, int state)
{
	struct cpu_info *ci = curcpu();

	KASSERT(ci->ci_ilevel == IPL_NONE);

	switch (method) {

	case ACPICPU_C_STATE_FFH:

		x86_monitor(&ci->ci_want_resched, 0, 0);

		if (__predict_false(ci->ci_want_resched != 0))
			return;

		x86_mwait((state - 1) << 4, 0);
		break;

	case ACPICPU_C_STATE_HALT:

		x86_disable_intr();

		if (__predict_false(ci->ci_want_resched != 0)) {
			x86_enable_intr();
			return;
		}

		x86_stihlt();
		break;
	}
}
Example #10
0
File: trap.c Project: MarginC/kame
static __inline void
userret (struct lwp *l, register_t pc, u_quad_t oticks)
{
	struct proc *p = l->l_proc;
	int sig;

	/* take pending signals */
	while ((sig = CURSIG(l)) != 0)
		postsig(sig);

	l->l_priority = l->l_usrpri;
	if (want_resched) {
		/*
		 * We're being preempted.
		 */
		preempt(0);
		while ((sig = CURSIG(l)) != 0)
			postsig(sig);
	}

	/*
	 * If profiling, charge recent system time to the trapped pc.
	 */
	if (l->l_flag & P_PROFIL) {
		extern int psratio;

		addupc_task(p, pc, (int)(p->p_sticks - oticks) * psratio);
	}

	curcpu()->ci_schedstate.spc_curpriority = l->l_priority;
}
Example #11
0
int
mtx_enter_try(struct mutex *mtx)
{
	struct cpu_info *owner, *ci = curcpu();
	int s;
	
 	if (mtx->mtx_wantipl != IPL_NONE)
		s = splraise(mtx->mtx_wantipl);

	owner = atomic_cas_ptr(&mtx->mtx_owner, NULL, ci);
#ifdef DIAGNOSTIC
	if (__predict_false(owner == ci))
		panic("mtx %p: locking against myself", mtx);
#endif
	if (owner == NULL) {
		if (mtx->mtx_wantipl != IPL_NONE)
			mtx->mtx_oldipl = s;
#ifdef DIAGNOSTIC
		ci->ci_mutex_level++;
#endif
		membar_enter();
		return (1);
	}

	if (mtx->mtx_wantipl != IPL_NONE)
		splx(s);

	return (0);
}
Example #12
0
void *
percpu_getref(percpu_t *pc)
{

	kpreempt_disable();
	return percpu_getptr_remote(pc, curcpu());
}
Example #13
0
/*
 * called once per clock tick via the pps callback
 * for the calibration of the TSC counters.
 * it is called only for the PRIMARY cpu. all
 * other cpus are called via a broadcast IPI
 * calibration interval is 1 second - we call
 * the calibration code only every hz calls
 */
static void
cc_calibrate(struct timecounter *tc)
{
	static int calls;
	struct cpu_info *ci;

	KASSERT(kpreempt_disabled());

	 /*
	  * XXX: for high interrupt frequency
	  * support: ++calls < hz / tc_tick
	  */
	if (++calls < hz)
		return;

	calls = 0;
	ci = curcpu();
	/* pick up reference ticks */
	cc_cal_val = cpu_counter32();

#if defined(MULTIPROCESSOR)
	cc_calibrate_mp(ci);
#endif
	cc_calibrate_cpu(ci);
}
Example #14
0
/*
 * Further secondary CPU initialization.
 *
 * We are now running on our startup stack, with proper page tables.
 * There is nothing to do but display some details about the CPU and its CMMUs.
 */
void
secondary_main()
{
	struct cpu_info *ci = curcpu();
	int s;

	cpu_configuration_print(0);
	ncpus++;

	sched_init_cpu(ci);
	nanouptime(&ci->ci_schedstate.spc_runtime);
	ci->ci_curproc = NULL;
	ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;

	/*
	 * Release cpu_hatch_mutex to let other secondary processors
	 * have a chance to run.
	 */
	hatch_pending_count--;
	__cpu_simple_unlock(&cpu_hatch_mutex);

	/* wait for cpu_boot_secondary_processors() */
	__cpu_simple_lock(&cpu_boot_mutex);
	__cpu_simple_unlock(&cpu_boot_mutex);

	spl0();
	SCHED_LOCK(s);
	set_psr(get_psr() & ~PSR_IND);

	SET(ci->ci_flags, CIF_ALIVE);

	cpu_switchto(NULL, sched_chooseproc());
}
Example #15
0
/*
 * Same as above, but also handles writeback completion on 68040.
 */
void
wb_userret(struct proc *p, struct frame *fp)
{
	int sig;
	union sigval sv;

	/* take pending signals */
	while ((sig = CURSIG(p)) != 0)
		postsig(sig);
	p->p_priority = p->p_usrpri;

	/*
	 * Deal with user mode writebacks (from trap, or from sigreturn).
	 * If any writeback fails, go back and attempt signal delivery
	 * unless we have already been here and attempted the writeback
	 * (e.g. bad address with user ignoring SIGSEGV).  In that case,
	 * we just return to the user without successfully completing
	 * the writebacks.  Maybe we should just drop the sucker?
	 */
	if (mmutype == MMU_68040 && fp->f_format == FMT7) {
		if ((sig = writeback(fp)) != 0) {
			sv.sival_ptr = (void *)fp->f_fmt7.f_fa;
			trapsignal(p, sig, T_MMUFLT, SEGV_MAPERR, sv);

			while ((sig = CURSIG(p)) != 0)
				postsig(sig);
			p->p_priority = p->p_usrpri;
		}
	}
	curcpu()->ci_schedstate.spc_curpriority = p->p_priority;
}
Example #16
0
void
cpu_intr(int ppl, vaddr_t pc, uint32_t status)
{
	struct cpu_info * const ci = curcpu();
	uint32_t pending;
	int ipl;
#ifdef DIAGNOSTIC
	const int mtx_count = ci->ci_mtx_count;
	const u_int biglock_count = ci->ci_biglock_count;
	const u_int blcnt = curlwp->l_blcnt;
#endif
	KASSERT(ci->ci_cpl == IPL_HIGH);
	KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);

	ci->ci_data.cpu_nintr++;

	while (ppl < (ipl = splintr(&pending))) {
		KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
		splx(ipl);	/* lower to interrupt level */
		KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);

		KASSERTMSG(ci->ci_cpl == ipl,
		    "%s: cpl (%d) != ipl (%d)", __func__, ci->ci_cpl, ipl);
		KASSERT(pending != 0);

		cf.pc = pc;
		cf.sr = status;
		cf.intr = (ci->ci_idepth > 1);

#ifdef MIPS3_ENABLE_CLOCK_INTR
		if (pending & MIPS_INT_MASK_5) {
			KASSERTMSG(ipl == IPL_SCHED,
			    "%s: ipl (%d) != IPL_SCHED (%d)",
			     __func__, ipl, IPL_SCHED);
			/* call the common MIPS3 clock interrupt handler */ 
			mips3_clockintr(&cf);
			pending ^= MIPS_INT_MASK_5;
		}
#endif

		if (pending != 0) {
			/* Process I/O and error interrupts. */
			evbmips_iointr(ipl, pc, pending);
		}
		KASSERT(biglock_count == ci->ci_biglock_count);
		KASSERT(blcnt == curlwp->l_blcnt);
		KASSERT(mtx_count == ci->ci_mtx_count);

		/*
		 * If even our spl is higher now (due to interrupting while
		 * spin-lock is held and higher IPL spin-lock is locked, it
		 * can no longer be locked so it's safe to lower IPL back
		 * to ppl.
		 */
		(void) splhigh();	/* disable interrupts */
	}

	KASSERT(ci->ci_cpl == IPL_HIGH);
	KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
}
Example #17
0
static void
amlogic_cpufreq_cb(void *arg1, void *arg2)
{
	struct cpu_info *ci = curcpu();

	ci->ci_data.cpu_cc_freq = cpufreq_get_rate() * 1000000;
}
Example #18
0
void
cpu_idle_mwait_cycle(void)
{
	struct cpu_info *ci = curcpu();

	if ((read_rflags() & PSL_I) == 0)
		panic("idle with interrupts blocked!");

	/* something already queued? */
	if (!cpu_is_idle(ci))
		return;

	/*
	 * About to idle; setting the MWAIT_IN_IDLE bit tells
	 * cpu_unidle() that it can't be a no-op and tells cpu_kick()
	 * that it doesn't need to use an IPI.  We also set the
	 * MWAIT_KEEP_IDLING bit: those routines clear it to stop
	 * the mwait.  Once they're set, we do a final check of the
	 * queue, in case another cpu called setrunqueue() and added
	 * something to the queue and called cpu_unidle() between
	 * the check in sched_idle() and here.
	 */
	atomic_setbits_int(&ci->ci_mwait, MWAIT_IDLING | MWAIT_ONLY);
	if (cpu_is_idle(ci)) {
		monitor(&ci->ci_mwait, 0, 0);
		if ((ci->ci_mwait & MWAIT_IDLING) == MWAIT_IDLING)
			mwait(0, 0);
	}

	/* done idling; let cpu_kick() know that an IPI is required */
	atomic_clearbits_int(&ci->ci_mwait, MWAIT_IDLING);
}
Example #19
0
void
__mp_lock(struct __mp_lock *mpl)
{
	int s;
	struct cpu_info *ci = curcpu();

	/*
	 * Please notice that mpl_count gets incremented twice for the
	 * first lock. This is on purpose. The way we release the lock
	 * in mp_unlock is to decrement the mpl_count and then check if
	 * the lock should be released. Since mpl_count is what we're
	 * spinning on, decrementing it in mpl_unlock to 0 means that
	 * we can't clear mpl_cpu, because we're no longer holding the
	 * lock. In theory mpl_cpu doesn't need to be cleared, but it's
	 * safer to clear it and besides, setting mpl_count to 2 on the
	 * first lock makes most of this code much simpler.
	 */
	while (1) {
		s = splhigh();
		if (__cpu_cas(&mpl->mpl_count, 0, 1) == 0) {
			alpha_mb();
			mpl->mpl_cpu = ci;
		}

		if (mpl->mpl_cpu == ci) {
			mpl->mpl_count++;
			splx(s);
			break;
		}
		splx(s);
		
		__mp_lock_spin(mpl);
	}
}
Example #20
0
static void
cal_timer(void)
{
	uint32_t cntfreq;

	cntfreq = curcpu()->ci_cpu_freq = RA_CLOCK_RATE;
	
	/* MIPS 4Kc CP0 counts every other clock */
	if (mips_options.mips_cpu_flags & CPU_MIPS_DOUBLE_COUNT)
		cntfreq /= 2;

	curcpu()->ci_cycles_per_hz = (cntfreq + hz / 2) / hz;

	/* Compute number of cycles per 1us (1/MHz). 0.5MHz is for roundup. */
	curcpu()->ci_divisor_delay = ((cntfreq + 500000) / 1000000);
}
Example #21
0
/*
 * cpuwatch_set - establish a MIPS COP0 watchpoint
 */
void
cpuwatch_set(cpu_watchpoint_t *cwp)
{
	struct cpu_info * const ci = curcpu();
	uint32_t watchhi;
	register_t watchlo;
	int cwnum = cwp - &ci->ci_cpuwatch_tab[0];

	KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] &&
		cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]);

	watchlo = cwp->cw_addr;
	if (cwp->cw_mode & CPUWATCH_WRITE)
		watchlo |= __BIT(0);
	if (cwp->cw_mode & CPUWATCH_READ)
		watchlo |= __BIT(1);
	if (cwp->cw_mode & CPUWATCH_EXEC)
		watchlo |= __BIT(2);

	if (cwp->cw_mode & CPUWATCH_ASID)
		watchhi = cwp->cw_asid << 16;	/* addr qualified by asid */
	else
		watchhi = __BIT(30);		/* addr not qual. by asid (Global) */
	if (cwp->cw_mode & CPUWATCH_MASK)
		watchhi |= cwp->cw_mask;	/* set "dont care" addr match bits */

	mipsNN_cp0_watchhi_write(cwnum, watchhi);
	mipsNN_cp0_watchlo_write(cwnum, watchlo);
}
Example #22
0
/*
 * Maskable IPIs.
 *
 * These IPIs are received as non maskable, but are not processed in
 * the NMI handler; instead, they are processed from the soft interrupt
 * handler.
 *
 * XXX This is grossly suboptimal.
 */
void
m197_soft_ipi()
{
	struct cpu_info *ci = curcpu();
	struct trapframe faketf;
	int s;

	__mp_lock(&kernel_lock);
	s = splclock();

	if (ci->ci_h_sxip != 0) {
		faketf.tf_cpu = ci;
		faketf.tf_sxip = ci->ci_h_sxip;
		faketf.tf_epsr = ci->ci_h_epsr;
		ci->ci_h_sxip = 0;
		hardclock((struct clockframe *)&faketf);
	}

	if (ci->ci_s_sxip != 0) {
		faketf.tf_cpu = ci;
		faketf.tf_sxip = ci->ci_s_sxip;
		faketf.tf_epsr = ci->ci_s_epsr;
		ci->ci_s_sxip = 0;
		statclock((struct clockframe *)&faketf);
	}

	splx(s);
	__mp_unlock(&kernel_lock);
}
Example #23
0
/*
 * Pause this cpu
 */
void
cpu_pause(struct reg *regsp)
{
	int s = splhigh();
	cpuid_t cii = cpu_index(curcpu());

	if (__predict_false(cold))
		return;

	do {
		kcpuset_atomic_set(cpus_paused, cii);
		do {
			;
		} while (kcpuset_isset(cpus_paused, cii));
		kcpuset_atomic_set(cpus_resumed, cii);
#if defined(DDB)
		if (ddb_running_on_this_cpu_p())
			cpu_Debugger();
		if (ddb_running_on_any_cpu_p())
			continue;
#endif
	} while (false);

	splx(s);
}
Example #24
0
void *
percpu_getref(percpu_t *pc)
{

    KPREEMPT_DISABLE(curlwp);
    return percpu_getptr_remote(pc, curcpu());
}
Example #25
0
/*
 * Independent profiling "tick" in case we're using a separate
 * clock or profiling event source.  Currently, that's just
 * performance counters--hence the wrapper.
 */
void
proftick(struct clockframe *frame)
{
#ifdef GPROF
        struct gmonparam *g;
        intptr_t i;
#endif
	struct lwp *l;
	struct proc *p;

	l = curcpu()->ci_data.cpu_onproc;
	p = (l ? l->l_proc : NULL);
	if (CLKF_USERMODE(frame)) {
		mutex_spin_enter(&p->p_stmutex);
		if (p->p_stflag & PST_PROFIL)
			addupc_intr(l, CLKF_PC(frame));
		mutex_spin_exit(&p->p_stmutex);
	} else {
#ifdef GPROF
		g = &_gmonparam;
		if (g->state == GMON_PROF_ON) {
			i = CLKF_PC(frame) - g->lowpc;
			if (i < g->textsize) {
				i /= HISTFRACTION * sizeof(*g->kcount);
				g->kcount[i]++;
			}
		}
#endif
#ifdef LWP_PC
		if (p != NULL && (p->p_stflag & PST_PROFIL) != 0)
			addupc_intr(l, LWP_PC(l));
#endif
	}
}
Example #26
0
void
userret(struct lwp *l)
{
#if defined(__PROG32) && defined(ARM_MMU_EXTENDED)
	/*
	 * If our ASID got released, access via TTBR0 will have been disabled.
	 * So if it is disabled, activate the lwp again to get a new ASID.
	 */
#ifdef __HAVE_PREEMPTION
	kpreempt_disable();
#endif
	KASSERT(curcpu()->ci_pmap_cur == l->l_proc->p_vmspace->vm_map.pmap);
	if (__predict_false(armreg_ttbcr_read() & TTBCR_S_PD0)) {
		pmap_activate(l);
	}
	KASSERT(!(armreg_ttbcr_read() & TTBCR_S_PD0));
#ifdef __HAVE_PREEMPTION
	kpreempt_enable();
#endif
#endif

	/* Invoke MI userret code */
	mi_userret(l);

#if defined(__PROG32) && defined(DIAGNOSTIC)
	KASSERT(VALID_R15_PSR(lwp_trapframe(l)->tf_pc,
	    lwp_trapframe(l)->tf_spsr));
#endif
}
Example #27
0
static void
openpic_send_ipi(cpuid_t target, uint32_t mesg)
{
	struct cpu_info * const ci = curcpu();
	uint32_t cpumask = 0;

	switch (target) {
		case IPI_DST_ALL:
		case IPI_DST_NOTME:
			for (u_int i = 0; i < ncpu; i++) {
				struct cpu_info * const dst_ci = cpu_lookup(i);
				if (target == IPI_DST_ALL || dst_ci != ci) {
					cpumask |= 1 << cpu_index(dst_ci);
					atomic_or_32(&dst_ci->ci_pending_ipis,
					    mesg);
				}
			}
			break;
		default: {
			struct cpu_info * const dst_ci = cpu_lookup(target);
			cpumask = 1 << cpu_index(dst_ci);
			atomic_or_32(&dst_ci->ci_pending_ipis, mesg);
			break;
		}
	}
	openpic_write(OPENPIC_IPI(cpu_index(ci), 1), cpumask);
}
Example #28
0
void
ampscu_attach(struct device *parent, struct device *self, void *args)
{
	struct ampscu_softc *sc = (struct ampscu_softc *)self;
	struct cortex_attach_args *ca = args;

	sc->sc_iot = ca->ca_iot;

	if (bus_space_map(sc->sc_iot, ca->ca_periphbase + SCU_ADDR,
	    SCU_SIZE, 0, &sc->sc_ioh))
		panic("ampscu_attach: bus_space_map failed!");

	ncpusfound = ampscu_ncpus(sc);

	printf(": %d CPUs\n", ncpusfound);

#ifdef MULTIPROCESSOR
	/* ARM Errata 764369 */
	if ((curcpu()->ci_arm_cpuid & CPU_ID_CORTEX_A9_MASK) == CPU_ID_CORTEX_A9)
	    bus_space_write_4(sc->sc_iot, sc->sc_ioh, 0x30,
		bus_space_read_4(sc->sc_iot, sc->sc_ioh, 0x30) | 1);

	bus_space_write_4(sc->sc_iot, sc->sc_ioh, SCU_CTRL,
	    bus_space_read_4(sc->sc_iot, sc->sc_ioh, SCU_CTRL) | 1);

	/* Flush ALL the caches. */
	cpu_drain_writebuf();
	cpu_idcache_wbinv_all();
	cpu_sdcache_wbinv_all();
	cpu_drain_writebuf();
#endif
}
/*
 * uvm_emap_switch: if the CPU is 'behind' the LWP in emap visibility,
 * perform TLB flush and thus update the local view.  Main purpose is
 * to handle kernel preemption, while emap is in use.
 *
 * => called from mi_switch(), when LWP returns after block or preempt.
 */
void
uvm_emap_switch(lwp_t *l)
{
	struct uvm_cpu *ucpu;
	u_int curgen, gen;

	KASSERT(kpreempt_disabled());

	/* If LWP did not use emap, then nothing to do. */
	if (__predict_true(l->l_emap_gen == UVM_EMAP_INACTIVE)) {
		return;
	}

	/*
	 * No need to synchronise if generation number of current CPU is
	 * newer than the number of this LWP.
	 *
	 * This test assumes two's complement arithmetic and allows
	 * ~2B missed updates before it will produce bad results.
	 */
	ucpu = curcpu()->ci_data.cpu_uvm;
	curgen = ucpu->emap_gen;
	gen = l->l_emap_gen;
	if (__predict_true((signed int)(curgen - gen) >= 0)) {
		return;
	}

	/*
	 * See comments in uvm_emap_consume() about memory
	 * barriers and race conditions.
	 */
	curgen = uvm_emap_gen_return();
	pmap_emap_sync(false);
	ucpu->emap_gen = curgen;
}
Example #30
0
/*
 * Send an interprocessor interrupt - sun4u.
 */
void
sparc64_send_ipi_sun4u(int upaid, ipifunc_t func, uint64_t arg1, uint64_t arg2)
{
	int i, ik, shift = 0;
	uint64_t intr_func;

	KASSERT(upaid != curcpu()->ci_cpuid);

	/*
	 * UltraSPARC-IIIi CPUs select the BUSY/NACK pair based on the
	 * lower two bits of the ITID.
	 */
	if (CPU_IS_USIIIi())
		shift = (upaid & 0x3) * 2;

	if (ldxa(0, ASI_IDSR) & (IDSR_BUSY << shift))
		panic("recursive IPI?");

	intr_func = (uint64_t)(u_long)func;

	/* Schedule an interrupt. */
	for (i = 0; i < 10000; i++) {
		int s = intr_disable();

		stxa(IDDR_0H, ASI_INTERRUPT_DISPATCH, intr_func);
		stxa(IDDR_1H, ASI_INTERRUPT_DISPATCH, arg1);
		stxa(IDDR_2H, ASI_INTERRUPT_DISPATCH, arg2);
		stxa(IDCR(upaid), ASI_INTERRUPT_DISPATCH, 0);
		membar_Sync();
		/* Workaround for SpitFire erratum #54, from FreeBSD */
		if (CPU_IS_SPITFIRE()) {
			(void)ldxa(P_DCR_0, ASI_INTERRUPT_RECEIVE_DATA);
			membar_Sync();
		}

		for (ik = 0; ik < 1000000; ik++) {
			if (ldxa(0, ASI_IDSR) & (IDSR_BUSY << shift))
				continue;
			else
				break;
		}
		intr_restore(s);

		if (ik == 1000000)
			break;

		if ((ldxa(0, ASI_IDSR) & (IDSR_NACK << shift)) == 0)
			return;
		/*
		 * Wait for a while with enabling interrupts to avoid
		 * deadlocks.  XXX - random value is better.
		 */
		DELAY(1);
	}

	if (panicstr == NULL)
		panic("cpu%d: ipi_send: couldn't send ipi to UPAID %u"
			" (tried %d times)", cpu_number(), upaid, i);
}