/*
* Place the thread in question on the run q.
*/
static void
shuttle_unsleep(kthread_t *t)
{
ASSERT(THREAD_LOCK_HELD(t));
/* Waiting on a shuttle */
ASSERT(t->t_wchan0 == (caddr_t)1 && t->t_wchan == NULL);
t->t_flag &= ~T_WAKEABLE;
t->t_wchan0 = NULL;
t->t_sobj_ops = NULL;
THREAD_TRANSITION(t);
CL_SETRUN(t);
}
/*
* Take thread off its wait queue and make it runnable.
* Returns with thread lock held.
*/
void
waitq_setrun(kthread_t *t)
{
waitq_t *wq = t->t_waitq;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(ISWAITING(t));
if (wq == NULL)
panic("waitq_setrun: thread %p is not on waitq", t);
waitq_dequeue(wq, t);
CL_SETRUN(t);
}
/*
* Change thread's priority while on the wait queue.
* Dequeue and equeue it again so that it gets placed in the right place.
*/
void
waitq_change_pri(kthread_t *t, pri_t new_pri)
{
waitq_t *wq = t->t_waitq;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(ISWAITING(t));
ASSERT(wq != NULL);
waitq_unlink(wq, t);
t->t_pri = new_pri;
waitq_link(wq, t);
}
/*
* Charge thread's project and return True if project or zone should be
* penalized because its project or zone is exceeding its cap. Also sets
* TS_PROJWAITQ or TS_ZONEWAITQ in this case.
*
* It is possible that the project cap is being disabled while this routine is
* executed. This should not cause any issues since the association between the
* thread and its project is protected by thread lock. It will still set
* TS_PROJECTWAITQ/TS_ZONEWAITQ in this case but cpucaps_enforce will not place
* anything on the blocked wait queue.
*
*/
boolean_t
cpucaps_charge(kthread_id_t t, caps_sc_t *csc, cpucaps_charge_t charge_type)
{
kproject_t *kpj = ttoproj(t);
klwp_t *lwp = t->t_lwp;
zone_t *zone;
cpucap_t *project_cap;
boolean_t rc = B_FALSE;
ASSERT(THREAD_LOCK_HELD(t));
/* Nothing to do for projects that are not capped. */
if (lwp == NULL || !PROJECT_IS_CAPPED(kpj))
return (B_FALSE);
caps_charge_adjust(t, csc);
/*
* The caller only requested to charge the project usage, no enforcement
* part.
*/
if (charge_type == CPUCAPS_CHARGE_ONLY)
return (B_FALSE);
project_cap = kpj->kpj_cpucap;
if (project_cap->cap_usage >= project_cap->cap_chk_value) {
t->t_schedflag |= TS_PROJWAITQ;
rc = B_TRUE;
} else if (t->t_schedflag & TS_PROJWAITQ) {
t->t_schedflag &= ~TS_PROJWAITQ;
}
zone = ttozone(t);
if (!ZONE_IS_CAPPED(zone)) {
if (t->t_schedflag & TS_ZONEWAITQ)
t->t_schedflag &= ~TS_ZONEWAITQ;
} else {
cpucap_t *zone_cap = zone->zone_cpucap;
if (zone_cap->cap_usage >= zone_cap->cap_chk_value) {
t->t_schedflag |= TS_ZONEWAITQ;
rc = B_TRUE;
} else if (t->t_schedflag & TS_ZONEWAITQ) {
t->t_schedflag &= ~TS_ZONEWAITQ;
}
}
return (rc);
}
/*
* Change the priority of a thread that's blocked on a condition variable.
*/
static void
cv_change_pri(kthread_t *t, pri_t pri, pri_t *t_prip)
{
condvar_impl_t *cvp = (condvar_impl_t *)t->t_wchan;
sleepq_t *sqp = t->t_sleepq;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(&SQHASH(cvp)->sq_queue == sqp);
if (cvp == NULL)
panic("cv_change_pri: %p not on sleep queue", t);
sleepq_dequeue(t);
*t_prip = pri;
sleepq_insert(sqp, t);
}
/*
* Charge project of thread t the time thread t spent on CPU since previously
* adjusted.
*
* Record the current on-CPU time in the csc structure.
*
* Do not adjust for more than one tick worth of time.
*
* It is possible that the project cap is being disabled while this routine is
* executed. This should not cause any issues since the association between the
* thread and its project is protected by thread lock.
*/
static void
caps_charge_adjust(kthread_id_t t, caps_sc_t *csc)
{
kproject_t *kpj = ttoproj(t);
hrtime_t new_usage;
hrtime_t usage_delta;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(kpj->kpj_cpucap != NULL);
/* Get on-CPU time since birth of a thread */
new_usage = mstate_thread_onproc_time(t);
/* Time spent on CPU since last checked */
usage_delta = new_usage - csc->csc_cputime;
/* Save the accumulated on-CPU time */
csc->csc_cputime = new_usage;
/* Charge at most one tick worth of on-CPU time */
if (usage_delta > cap_tick_cost)
usage_delta = cap_tick_cost;
/* Add usage_delta to the project usage value. */
if (usage_delta > 0) {
cpucap_t *cap = kpj->kpj_cpucap;
DTRACE_PROBE2(cpucaps__project__charge,
kthread_id_t, t, hrtime_t, usage_delta);
disp_lock_enter_high(&cap->cap_usagelock);
cap->cap_usage += usage_delta;
/* Check for overflows */
if (cap->cap_usage < 0)
cap->cap_usage = MAX_USAGE - 1;
disp_lock_exit_high(&cap->cap_usagelock);
/*
* cap_maxusage is only kept for observability. Move it outside
* the lock to reduce the time spent while holding the lock.
*/
if (cap->cap_usage > cap->cap_maxusage)
cap->cap_maxusage = cap->cap_usage;
}
}
static void
waitq_dequeue(waitq_t *wq, kthread_t *t)
{
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(t->t_waitq == wq);
ASSERT(ISWAITING(t));
waitq_unlink(wq, t);
DTRACE_SCHED1(cpucaps__wakeup, kthread_t *, t);
/*
* Change thread to transition state and drop the wait queue lock. The
* thread will remain locked since its t_lockp points to the
* transition_lock.
*/
THREAD_TRANSITION(t);
}
/*
* Unsleep a thread that's blocked on a condition variable.
*/
static void
cv_unsleep(kthread_t *t)
{
condvar_impl_t *cvp = (condvar_impl_t *)t->t_wchan;
sleepq_head_t *sqh = SQHASH(cvp);
ASSERT(THREAD_LOCK_HELD(t));
if (cvp == NULL)
panic("cv_unsleep: thread %p not on sleepq %p", t, sqh);
DTRACE_SCHED1(wakeup, kthread_t *, t);
sleepq_unsleep(t);
if (cvp->cv_waiters != CV_MAX_WAITERS)
cvp->cv_waiters--;
disp_lock_exit_high(&sqh->sq_lock);
CL_SETRUN(t);
}
/*
* Return the amount of onproc and runnable time this thread has experienced.
*
* Because the fields we read are not protected by locks when updated
* by the thread itself, this is an inherently racey interface. In
* particular, the ASSERT(THREAD_LOCK_HELD(t)) doesn't guarantee as much
* as it might appear to.
*
* The implication for users of this interface is that onproc and runnable
* are *NOT* monotonically increasing; they may temporarily be larger than
* they should be.
*/
void
mstate_systhread_times(kthread_t *t, hrtime_t *onproc, hrtime_t *runnable)
{
struct mstate *const ms = &ttolwp(t)->lwp_mstate;
int mstate;
hrtime_t now;
hrtime_t state_start;
hrtime_t waitrq;
hrtime_t aggr_onp;
hrtime_t aggr_run;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(t->t_procp->p_flag & SSYS);
ASSERT(ttolwp(t) != NULL);
/* shouldn't be any non-SYSTEM on-CPU time */
ASSERT(ms->ms_acct[LMS_USER] == 0);
ASSERT(ms->ms_acct[LMS_TRAP] == 0);
mstate = t->t_mstate;
waitrq = t->t_waitrq;
state_start = ms->ms_state_start;
aggr_onp = ms->ms_acct[LMS_SYSTEM];
aggr_run = ms->ms_acct[LMS_WAIT_CPU];
now = gethrtime_unscaled();
/* if waitrq == 0, then there is no time to account to TS_RUN */
if (waitrq == 0)
waitrq = now;
/* If there is system time to accumulate, do so */
if (mstate == LMS_SYSTEM && state_start < waitrq)
aggr_onp += waitrq - state_start;
if (waitrq < now)
aggr_run += now - waitrq;
scalehrtime(&aggr_onp);
scalehrtime(&aggr_run);
*onproc = aggr_onp;
*runnable = aggr_run;
}
/*
* Take the first thread off the wait queue and make it runnable.
* Return the pointer to the thread or NULL if waitq is empty
*/
static kthread_t *
waitq_runfirst(waitq_t *wq)
{
kthread_t *t;
t = waitq_takeone(wq);
if (t != NULL) {
/*
* t should have transition lock held.
* CL_SETRUN() will replace it with dispq lock and keep it held.
* thread_unlock() will drop dispq lock and restore PIL.
*/
ASSERT(THREAD_LOCK_HELD(t));
CL_SETRUN(t);
thread_unlock(t);
}
return (t);
}
/*
* Return an aggregation of user and system CPU time consumed by
* the specified thread in scaled nanoseconds.
*/
hrtime_t
mstate_thread_onproc_time(kthread_t *t)
{
hrtime_t aggr_time;
hrtime_t now;
hrtime_t waitrq;
hrtime_t state_start;
struct mstate *ms;
klwp_t *lwp;
int mstate;
ASSERT(THREAD_LOCK_HELD(t));
if ((lwp = ttolwp(t)) == NULL)
return (0);
mstate = t->t_mstate;
waitrq = t->t_waitrq;
ms = &lwp->lwp_mstate;
state_start = ms->ms_state_start;
aggr_time = ms->ms_acct[LMS_USER] +
ms->ms_acct[LMS_SYSTEM] + ms->ms_acct[LMS_TRAP];
now = gethrtime_unscaled();
/*
* NOTE: gethrtime_unscaled on X86 taken on different CPUs is
* inconsistent, so it is possible that now < state_start.
*/
if (mstate == LMS_USER || mstate == LMS_SYSTEM || mstate == LMS_TRAP) {
/* if waitrq is zero, count all of the time. */
if (waitrq == 0) {
waitrq = now;
}
if (waitrq > state_start) {
aggr_time += waitrq - state_start;
}
}
scalehrtime(&aggr_time);
return (aggr_time);
}
static void
waitq_unlink(waitq_t *wq, kthread_t *t)
{
kthread_t *nt;
kthread_t **ptl;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(DISP_LOCK_HELD(&wq->wq_lock));
ASSERT(t->t_waitq == wq);
ptl = &t->t_priback->t_link;
/*
* Is it the head of a priority sublist? If so, need to walk
* the priorities to find the t_link pointer that points to it.
*/
if (*ptl != t) {
/*
* Find the right priority level.
*/
ptl = &t->t_waitq->wq_first;
while ((nt = *ptl) != t)
ptl = &nt->t_priback->t_link;
}
/*
* Remove thread from the t_link list.
*/
*ptl = t->t_link;
/*
* Take it off the priority sublist if there's more than one
* thread there.
*/
if (t->t_priforw != t) {
t->t_priback->t_priforw = t->t_priforw;
t->t_priforw->t_priback = t->t_priback;
}
t->t_link = NULL;
wq->wq_count--;
t->t_waitq = NULL;
t->t_priforw = NULL;
t->t_priback = NULL;
}
/*
* Remove a thread from the sleep queue for this
* semaphore.
*/
static void
sema_dequeue(ksema_t *sp, kthread_t *t)
{
kthread_t **tpp;
kthread_t *tp;
sema_impl_t *s;
ASSERT(THREAD_LOCK_HELD(t));
s = (sema_impl_t *)sp;
tpp = &s->s_slpq;
while ((tp = *tpp) != NULL) {
if (tp == t) {
*tpp = t->t_link;
t->t_link = NULL;
return;
}
tpp = &tp->t_link;
}
}
/*
* The cv_block() function blocks a thread on a condition variable
* by putting it in a hashed sleep queue associated with the
* synchronization object.
*
* Threads are taken off the hashed sleep queues via calls to
* cv_signal(), cv_broadcast(), or cv_unsleep().
*/
static void
cv_block(condvar_impl_t *cvp)
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
sleepq_head_t *sqh;
ASSERT(THREAD_LOCK_HELD(t));
ASSERT(t != CPU->cpu_idle_thread);
ASSERT(CPU_ON_INTR(CPU) == 0);
ASSERT(t->t_wchan0 == NULL && t->t_wchan == NULL);
ASSERT(t->t_state == TS_ONPROC);
t->t_schedflag &= ~TS_SIGNALLED;
CL_SLEEP(t); /* assign kernel priority */
t->t_wchan = (caddr_t)cvp;
t->t_sobj_ops = &cv_sobj_ops;
DTRACE_SCHED(sleep);
/*
* The check for t_intr is to avoid doing the
* account for an interrupt thread on the still-pinned
* lwp's statistics.
*/
if (lwp != NULL && t->t_intr == NULL) {
lwp->lwp_ru.nvcsw++;
(void) new_mstate(t, LMS_SLEEP);
}
sqh = SQHASH(cvp);
disp_lock_enter_high(&sqh->sq_lock);
if (cvp->cv_waiters < CV_MAX_WAITERS)
cvp->cv_waiters++;
ASSERT(cvp->cv_waiters <= CV_MAX_WAITERS);
THREAD_SLEEP(t, &sqh->sq_lock);
sleepq_insert(&sqh->sq_queue, t);
/*
* THREAD_SLEEP() moves curthread->t_lockp to point to the
* lock sqh->sq_lock. This lock is later released by the caller
* when it calls thread_unlock() on curthread.
*/
}
/*
* Put a thread on the sleep queue for this semaphore.
*/
static void
sema_queue(ksema_t *sp, kthread_t *t)
{
kthread_t **tpp;
kthread_t *tp;
pri_t cpri;
sema_impl_t *s;
ASSERT(THREAD_LOCK_HELD(t));
s = (sema_impl_t *)sp;
tpp = &s->s_slpq;
cpri = DISP_PRIO(t);
while ((tp = *tpp) != NULL) {
if (cpri > DISP_PRIO(tp))
break;
tpp = &tp->t_link;
}
*tpp = t;
t->t_link = tp;
}
/*
* Check if thread can be moved to a new cpu partition. Called by
* cpupart_move_thread() and pset_bind_start().
*/
int
cpupart_movable_thread(kthread_id_t tp, cpupart_t *cp, int ignore)
{
ASSERT(MUTEX_HELD(&cpu_lock));
ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
ASSERT(cp != NULL);
ASSERT(THREAD_LOCK_HELD(tp));
/*
* CPU-bound threads can't be moved.
*/
if (!ignore) {
cpu_t *boundcpu = tp->t_bound_cpu ? tp->t_bound_cpu :
tp->t_weakbound_cpu;
if (boundcpu != NULL && boundcpu->cpu_part != cp)
return (EBUSY);
}
if (tp->t_cid == sysdccid) {
return (EINVAL); /* For now, sysdc threads can't move */
}
return (0);
}
/*ARGSUSED*/
static void
shuttle_change_pri(kthread_t *t, pri_t p, pri_t *t_prip)
{
ASSERT(THREAD_LOCK_HELD(t));
*t_prip = p;
}
/*
* Sets the value of the yield field for the specified thread.
* Called by ts_preempt() and ts_tick() to set the field, and
* ts_yield() to clear it.
* The kernel never looks at this field so we don't need a
* schedctl_get_yield() function.
*/
void
schedctl_set_yield(kthread_t *t, short val)
{
ASSERT(THREAD_LOCK_HELD(t));
t->t_schedctl->sc_preemptctl.sc_yield = val;
}
/*
* similiar to sema_p except that it blocks at an interruptible
* priority. if a signal is present then return 1 otherwise 0.
*/
int
sema_p_sig(ksema_t *sp)
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
sema_impl_t *s;
disp_lock_t *sqlp;
if (lwp == NULL) {
sema_p(sp);
return (0);
}
s = (sema_impl_t *)sp;
sqlp = &SQHASH(s)->sq_lock;
disp_lock_enter(sqlp);
ASSERT(s->s_count >= 0);
while (s->s_count == 0) {
proc_t *p = ttoproc(t);
thread_lock_high(t);
t->t_flag |= T_WAKEABLE;
SEMA_BLOCK(s, sqlp);
lwp->lwp_asleep = 1;
lwp->lwp_sysabort = 0;
thread_unlock_nopreempt(t);
if (ISSIG(t, JUSTLOOKING) || MUSTRETURN(p, t))
setrun(t);
swtch();
t->t_flag &= ~T_WAKEABLE;
if (ISSIG(t, FORREAL) ||
lwp->lwp_sysabort || MUSTRETURN(p, t)) {
kthread_t *sq, *tp;
lwp->lwp_asleep = 0;
lwp->lwp_sysabort = 0;
disp_lock_enter(sqlp);
sq = s->s_slpq;
/*
* in case sema_v and interrupt happen
* at the same time, we need to pass the
* sema_v to the next thread.
*/
if ((sq != NULL) && (s->s_count > 0)) {
tp = sq;
ASSERT(THREAD_LOCK_HELD(tp));
sq = sq->t_link;
tp->t_link = NULL;
DTRACE_SCHED1(wakeup, kthread_t *, tp);
tp->t_sobj_ops = NULL;
tp->t_wchan = NULL;
ASSERT(tp->t_state == TS_SLEEP);
CL_WAKEUP(tp);
s->s_slpq = sq;
disp_lock_exit_high(sqlp);
thread_unlock(tp);
} else {
disp_lock_exit(sqlp);
}
return (1);
}
lwp->lwp_asleep = 0;
disp_lock_enter(sqlp);
}
s->s_count--;
disp_lock_exit(sqlp);
return (0);
}
/*
* Returns non-zero if the specified thread shouldn't be preempted at this time.
* Called by ts_preempt(), ts_tick(), and ts_update().
*/
int
schedctl_get_nopreempt(kthread_t *t)
{
ASSERT(THREAD_LOCK_HELD(t));
return (t->t_schedctl->sc_preemptctl.sc_nopreempt);
}
