/* * Mark the current thread as sleeping on a shuttle object, and * switch to a new thread. * No locks other than 'l' should be held at this point. */ void shuttle_swtch(kmutex_t *l) { klwp_t *lwp = ttolwp(curthread); thread_lock(curthread); disp_lock_enter_high(&shuttle_lock); lwp->lwp_asleep = 1; /* /proc */ lwp->lwp_sysabort = 0; /* /proc */ lwp->lwp_ru.nvcsw++; curthread->t_flag |= T_WAKEABLE; curthread->t_sobj_ops = &shuttle_sobj_ops; curthread->t_wchan0 = (caddr_t)1; CL_INACTIVE(curthread); DTRACE_SCHED(sleep); THREAD_SLEEP(curthread, &shuttle_lock); (void) new_mstate(curthread, LMS_SLEEP); disp_lock_exit_high(&shuttle_lock); mutex_exit(l); if (ISSIG(curthread, JUSTLOOKING) || MUSTRETURN(curproc, curthread)) setrun(curthread); swtch(); /* * Caller must check for ISSIG/lwp_sysabort conditions * and clear lwp->lwp_asleep/lwp->lwp_sysabort */ }
/* * The cv_block() function blocks a thread on a condition variable * by putting it in a hashed sleep queue associated with the * synchronization object. * * Threads are taken off the hashed sleep queues via calls to * cv_signal(), cv_broadcast(), or cv_unsleep(). */ static void cv_block(condvar_impl_t *cvp) { kthread_t *t = curthread; klwp_t *lwp = ttolwp(t); sleepq_head_t *sqh; ASSERT(THREAD_LOCK_HELD(t)); ASSERT(t != CPU->cpu_idle_thread); ASSERT(CPU_ON_INTR(CPU) == 0); ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL); ASSERT(t->t_state == TS_ONPROC); t->t_schedflag &= ~TS_SIGNALLED; CL_SLEEP(t); /* assign kernel priority */ t->t_wchan = (caddr_t)cvp; t->t_sobj_ops = &cv_sobj_ops; DTRACE_SCHED(sleep); /* * The check for t_intr is to avoid doing the * account for an interrupt thread on the still-pinned * lwp's statistics. */ if (lwp != NULL && t->t_intr == NULL) { lwp->lwp_ru.nvcsw++; (void) new_mstate(t, LMS_SLEEP); } sqh = SQHASH(cvp); disp_lock_enter_high(&sqh->sq_lock); if (cvp->cv_waiters < CV_MAX_WAITERS) cvp->cv_waiters++; ASSERT(cvp->cv_waiters <= CV_MAX_WAITERS); THREAD_SLEEP(t, &sqh->sq_lock); sleepq_insert(&sqh->sq_queue, t); /* * THREAD_SLEEP() moves curthread->t_lockp to point to the * lock sqh->sq_lock. This lock is later released by the caller * when it calls thread_unlock() on curthread. */ }
void fp_precise(struct regs *rp) { fp_simd_type fpsd; int inst_ftt; union { uint_t i; fp_inst_type inst; } kluge; klwp_t *lwp = ttolwp(curthread); kfpu_t *fp = lwptofpu(lwp); uint64_t gsr; int mstate; if (fpu_exists) save_gsr(fp); gsr = get_gsr(fp); /* * Get the instruction to be emulated from the pc saved by the trap. * Note that the kernel is NOT prepared to handle a kernel fp * exception if it can't pass successfully through the fp simulator. * * If the trap occurred in user mode, set lwp_state to LWP_SYS for the * purposes of clock accounting and switch to the LMS_TRAP microstate. */ if (USERMODE(rp->r_tstate)) { inst_ftt = _fp_read_inst((uint32_t *)rp->r_pc, &kluge.i, &fpsd); mstate = new_mstate(curthread, LMS_TRAP); lwp->lwp_state = LWP_SYS; } else { kluge.i = *(uint_t *)rp->r_pc; inst_ftt = ftt_none; } if (inst_ftt != ftt_none) { /* * Save the bad address and post the signal. * It can only be an ftt_alignment or ftt_fault trap. * XXX - How can this work w/mainsail and do_unaligned? */ fpsd.fp_trapaddr = (caddr_t)rp->r_pc; fp_traps(&fpsd, inst_ftt, rp); } else { /* * Conjure up a floating point queue and advance the pc/npc * to fake a deferred fp trap. We now run the fp simulator * in fp_precise, while allowing setfpregs to call fp_runq, * because this allows us to do the ugly machinations to * inc/dec the pc depending on the trap type, as per * bugid 1210159. fp_runq is still going to have the * generic "how do I connect the "fp queue to the pc/npc" * problem alluded to in bugid 1192883, which is only a * problem for a restorecontext of a v8 fp queue on a * v9 system, which seems like the .000000001% case (on v9)! */ struct _fpq *pfpq = &fp->fpu_q->FQu.fpq; fp_simd_type fpsd; int fptrap; pfpq->fpq_addr = (uint_t *)rp->r_pc; pfpq->fpq_instr = kluge.i; fp->fpu_qcnt = 1; fp->fpu_q_entrysize = sizeof (struct _fpq); kpreempt_disable(); (void) flush_user_windows_to_stack(NULL); fptrap = fpu_vis_sim((fp_simd_type *)&fpsd, (fp_inst_type *)pfpq->fpq_addr, rp, (fsr_type *)&fp->fpu_fsr, gsr, kluge.i); /* update the hardware fp fsr state for sake of ucontext */ if (fpu_exists) _fp_write_pfsr(&fp->fpu_fsr); if (fptrap) { /* back up the pc if the signal needs to be precise */ if (fptrap != ftt_ieee) { fp->fpu_qcnt = 0; } /* post signal */ fp_traps(&fpsd, fptrap, rp); /* decrement queue count for ieee exceptions */ if (fptrap == ftt_ieee) { fp->fpu_qcnt = 0; } } else { fp->fpu_qcnt = 0; } /* update the software pcb copies of hardware fp registers */ if (fpu_exists) { fp_save(fp); } kpreempt_enable(); } /* * Reset lwp_state to LWP_USER for the purposes of clock accounting, * and restore the previously saved microstate. */ if (USERMODE(rp->r_tstate)) { (void) new_mstate(curthread, mstate); lwp->lwp_state = LWP_USER; } }
int turnstile_block(turnstile_t *ts, int qnum, void *sobj, sobj_ops_t *sobj_ops, kmutex_t *mp, lwp_timer_t *lwptp) { kthread_t *owner; kthread_t *t = curthread; proc_t *p = ttoproc(t); klwp_t *lwp = ttolwp(t); turnstile_chain_t *tc = &TURNSTILE_CHAIN(sobj); int error = 0; int loser = 0; ASSERT(DISP_LOCK_HELD(&tc->tc_lock)); ASSERT(mp == NULL || IS_UPI(mp)); ASSERT((SOBJ_TYPE(sobj_ops) == SOBJ_USER_PI) ^ (mp == NULL)); thread_lock_high(t); if (ts == NULL) { /* * This is the first thread to block on this sobj. * Take its attached turnstile and add it to the hash chain. */ ts = t->t_ts; ts->ts_sobj = sobj; ts->ts_next = tc->tc_first; tc->tc_first = ts; ASSERT(ts->ts_waiters == 0); } else { /* * Another thread has already donated its turnstile * to block on this sobj, so ours isn't needed. * Stash it on the active turnstile's freelist. */ turnstile_t *myts = t->t_ts; myts->ts_free = ts->ts_free; ts->ts_free = myts; t->t_ts = ts; ASSERT(ts->ts_sobj == sobj); ASSERT(ts->ts_waiters > 0); } /* * Put the thread to sleep. */ ASSERT(t != CPU->cpu_idle_thread); ASSERT(CPU_ON_INTR(CPU) == 0); ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL); ASSERT(t->t_state == TS_ONPROC); if (SOBJ_TYPE(sobj_ops) == SOBJ_USER_PI) { curthread->t_flag |= T_WAKEABLE; } CL_SLEEP(t); /* assign kernel priority */ THREAD_SLEEP(t, &tc->tc_lock); t->t_wchan = sobj; t->t_sobj_ops = sobj_ops; DTRACE_SCHED(sleep); if (lwp != NULL) { lwp->lwp_ru.nvcsw++; (void) new_mstate(t, LMS_SLEEP); if (SOBJ_TYPE(sobj_ops) == SOBJ_USER_PI) { lwp->lwp_asleep = 1; lwp->lwp_sysabort = 0; /* * make wchan0 non-zero to conform to the rule that * threads blocking for user-level objects have a * non-zero wchan0: this prevents spurious wake-ups * by, for example, /proc. */ t->t_wchan0 = (caddr_t)1; } } ts->ts_waiters++; sleepq_insert(&ts->ts_sleepq[qnum], t); if (SOBJ_TYPE(sobj_ops) == SOBJ_MUTEX && SOBJ_OWNER(sobj_ops, sobj) == NULL) panic("turnstile_block(%p): unowned mutex", (void *)ts); /* * Follow the blocking chain to its end, willing our priority to * everyone who's in our way. */ while (t->t_sobj_ops != NULL && (owner = SOBJ_OWNER(t->t_sobj_ops, t->t_wchan)) != NULL) { if (owner == curthread) { if (SOBJ_TYPE(sobj_ops) != SOBJ_USER_PI) { panic("Deadlock: cycle in blocking chain"); } /* * If the cycle we've encountered ends in mp, * then we know it isn't a 'real' cycle because * we're going to drop mp before we go to sleep. * Moreover, since we've come full circle we know * that we must have willed priority to everyone * in our way. Therefore, we can break out now. */ if (t->t_wchan == (void *)mp) break; if (loser) lock_clear(&turnstile_loser_lock); /* * For SOBJ_USER_PI, a cycle is an application * deadlock which needs to be communicated * back to the application. */ thread_unlock_nopreempt(t); mutex_exit(mp); setrun(curthread); swtch(); /* necessary to transition state */ curthread->t_flag &= ~T_WAKEABLE; if (lwptp->lwpt_id != 0) (void) lwp_timer_dequeue(lwptp); setallwatch(); lwp->lwp_asleep = 0; lwp->lwp_sysabort = 0; return (EDEADLK); } if (!turnstile_interlock(t->t_lockp, &owner->t_lockp)) { /* * If we failed to grab the owner's thread lock, * turnstile_interlock() will have dropped t's * thread lock, so at this point we don't even know * that 't' exists anymore. The simplest solution * is to restart the entire priority inheritance dance * from the beginning of the blocking chain, since * we *do* know that 'curthread' still exists. * Application of priority inheritance is idempotent, * so it's OK that we're doing it more than once. * Note also that since we've dropped our thread lock, * we may already have been woken up; if so, our * t_sobj_ops will be NULL, the loop will terminate, * and the call to swtch() will be a no-op. Phew. * * There is one further complication: if two (or more) * threads keep trying to grab the turnstile locks out * of order and keep losing the race to another thread, * these "dueling losers" can livelock the system. * Therefore, once we get into this rare situation, * we serialize all the losers. */ if (loser == 0) { loser = 1; lock_set(&turnstile_loser_lock); } t = curthread; thread_lock_high(t); continue; } /* * We now have the owner's thread lock. If we are traversing * from non-SOBJ_USER_PI ops to SOBJ_USER_PI ops, then we know * that we have caught the thread while in the TS_SLEEP state, * but holding mp. We know that this situation is transient * (mp will be dropped before the holder actually sleeps on * the SOBJ_USER_PI sobj), so we will spin waiting for mp to * be dropped. Then, as in the turnstile_interlock() failure * case, we will restart the priority inheritance dance. */ if (SOBJ_TYPE(t->t_sobj_ops) != SOBJ_USER_PI && owner->t_sobj_ops != NULL && SOBJ_TYPE(owner->t_sobj_ops) == SOBJ_USER_PI) { kmutex_t *upi_lock = (kmutex_t *)t->t_wchan; ASSERT(IS_UPI(upi_lock)); ASSERT(SOBJ_TYPE(t->t_sobj_ops) == SOBJ_MUTEX); if (t->t_lockp != owner->t_lockp) thread_unlock_high(owner); thread_unlock_high(t); if (loser) lock_clear(&turnstile_loser_lock); while (mutex_owner(upi_lock) == owner) { SMT_PAUSE(); continue; } if (loser) lock_set(&turnstile_loser_lock); t = curthread; thread_lock_high(t); continue; } turnstile_pi_inherit(t->t_ts, owner, DISP_PRIO(t)); if (t->t_lockp != owner->t_lockp) thread_unlock_high(t); t = owner; } if (loser) lock_clear(&turnstile_loser_lock); /* * Note: 't' and 'curthread' were synonymous before the loop above, * but now they may be different. ('t' is now the last thread in * the blocking chain.) */ if (SOBJ_TYPE(sobj_ops) == SOBJ_USER_PI) { ushort_t s = curthread->t_oldspl; int timedwait = 0; uint_t imm_timeout = 0; clock_t tim = -1; thread_unlock_high(t); if (lwptp->lwpt_id != 0) { /* * We enqueued a timeout. If it has already fired, * lwptp->lwpt_imm_timeout has been set with cas, * so fetch it with cas. */ timedwait = 1; imm_timeout = atomic_cas_uint(&lwptp->lwpt_imm_timeout, 0, 0); } mutex_exit(mp); splx(s); if (ISSIG(curthread, JUSTLOOKING) || MUSTRETURN(p, curthread) || imm_timeout) setrun(curthread); swtch(); curthread->t_flag &= ~T_WAKEABLE; if (timedwait) tim = lwp_timer_dequeue(lwptp); setallwatch(); if (ISSIG(curthread, FORREAL) || lwp->lwp_sysabort || MUSTRETURN(p, curthread)) error = EINTR; else if (imm_timeout || (timedwait && tim == -1)) error = ETIME; lwp->lwp_sysabort = 0; lwp->lwp_asleep = 0; } else { thread_unlock_nopreempt(t); swtch(); } return (error); }
/* * Mark the current thread as sleeping on a shuttle object, and * resume the specified thread. The 't' thread must be marked as ONPROC. * * No locks other than 'l' should be held at this point. */ void shuttle_resume(kthread_t *t, kmutex_t *l) { klwp_t *lwp = ttolwp(curthread); cpu_t *cp; disp_lock_t *oldtlp; thread_lock(curthread); disp_lock_enter_high(&shuttle_lock); if (lwp != NULL) { lwp->lwp_asleep = 1; /* /proc */ lwp->lwp_sysabort = 0; /* /proc */ lwp->lwp_ru.nvcsw++; } curthread->t_flag |= T_WAKEABLE; curthread->t_sobj_ops = &shuttle_sobj_ops; /* * setting cpu_dispthread before changing thread state * so that kernel preemption will be deferred to after swtch_to() */ cp = CPU; cp->cpu_dispthread = t; cp->cpu_dispatch_pri = DISP_PRIO(t); /* * Set the wchan0 field so that /proc won't just do a setrun * on this thread when trying to stop a process. Instead, * /proc will mark the thread as VSTOPPED similar to threads * that are blocked on user level condition variables. */ curthread->t_wchan0 = (caddr_t)1; CL_INACTIVE(curthread); DTRACE_SCHED1(wakeup, kthread_t *, t); DTRACE_SCHED(sleep); THREAD_SLEEP(curthread, &shuttle_lock); disp_lock_exit_high(&shuttle_lock); /* * Update ustate records (there is no waitrq obviously) */ (void) new_mstate(curthread, LMS_SLEEP); thread_lock_high(t); oldtlp = t->t_lockp; restore_mstate(t); t->t_flag &= ~T_WAKEABLE; t->t_wchan0 = NULL; t->t_sobj_ops = NULL; /* * Make sure we end up on the right CPU if we are dealing with bound * CPU's or processor partitions. */ if (t->t_bound_cpu != NULL || t->t_cpupart != cp->cpu_part) { aston(t); cp->cpu_runrun = 1; } /* * We re-assign t_disp_queue and t_lockp of 't' here because * 't' could have been preempted. */ if (t->t_disp_queue != cp->cpu_disp) { t->t_disp_queue = cp->cpu_disp; thread_onproc(t, cp); } /* * We can't call thread_unlock_high() here because t's thread lock * could have changed by thread_onproc() call above to point to * CPU->cpu_thread_lock. */ disp_lock_exit_high(oldtlp); mutex_exit(l); /* * Make sure we didn't receive any important events while * we weren't looking */ if (lwp && (ISSIG(curthread, JUSTLOOKING) || MUSTRETURN(curproc, curthread))) setrun(curthread); swtch_to(t); /* * Caller must check for ISSIG/lwp_sysabort conditions * and clear lwp->lwp_asleep/lwp->lwp_sysabort */ }