int
_pthread_spin_lock(pthread_spinlock_t *lock)
{
struct pthread_spinlock *lck;
struct pthread *self = _pthread_self();
int count, oldval, ret;
if (lock == NULL || (lck = *lock) == NULL)
ret = EINVAL;
else if (lck->s_owner == self)
ret = EDEADLK;
else {
do {
count = SPIN_COUNT;
while (lck->s_lock) {
#ifdef __i386__
/* tell cpu we are spinning */
__asm __volatile("pause");
#endif
if (--count <= 0) {
count = SPIN_COUNT;
_pthread_yield();
}
}
atomic_swap_int(&(lck)->s_lock, 1, &oldval);
} while (oldval);
lck->s_owner = self;
ret = 0;
}
return (ret);
}
/*
* IPIs are 'fast' interrupts, so we deal with them directly from our
* signal handler.
*
* WARNING: Signals are not physically disabled here so we have to enter
* our critical section before bumping gd_intr_nesting_level or another
* interrupt can come along and get really confused.
*/
static
void
ipisig(int nada, siginfo_t *info, void *ctxp)
{
globaldata_t gd = mycpu;
thread_t td = gd->gd_curthread;
if (td->td_critcount == 0) {
++td->td_critcount;
++gd->gd_intr_nesting_level;
atomic_swap_int(&gd->gd_npoll, 0);
lwkt_process_ipiq();
--gd->gd_intr_nesting_level;
--td->td_critcount;
} else {
need_ipiq();
}
}
int
_pthread_spin_unlock(pthread_spinlock_t *lock)
{
struct pthread_spinlock *lck;
int ret;
if (lock == NULL || (lck = *lock) == NULL)
ret = EINVAL;
else {
if (lck->s_owner != _pthread_self())
ret = EPERM;
else {
lck->s_owner = NULL;
atomic_swap_int(&lck->s_lock, 0, &ret);
ret = 0;
}
}
return (ret);
}
/*
* IPIs are 'fast' interrupts, so we deal with them directly from our
* signal handler.
*
* WARNING: Signals are not physically disabled here so we have to enter
* our critical section before bumping gd_intr_nesting_level or another
* interrupt can come along and get really confused.
*/
static
void
ipisig(int nada, siginfo_t *info, void *ctxp)
{
globaldata_t gd = mycpu;
thread_t td = gd->gd_curthread;
int save;
save = errno;
if (td->td_critcount == 0) {
crit_enter_raw(td);
++gd->gd_cnt.v_ipi;
++gd->gd_intr_nesting_level;
atomic_swap_int(&gd->gd_npoll, 0);
lwkt_process_ipiq();
--gd->gd_intr_nesting_level;
crit_exit_raw(td);
} else {
need_ipiq();
}
errno = save;
}
static int
acpi_cst_set_lowest_oncpu(struct acpi_cst_softc *sc, int val)
{
int old_lowest, error = 0, old_lowest_req;
uint32_t old_type, type;
KKASSERT(mycpuid == sc->cst_cpuid);
old_lowest_req = sc->cst_cx_lowest_req;
sc->cst_cx_lowest_req = val;
if (val > sc->cst_cx_count - 1)
val = sc->cst_cx_count - 1;
old_lowest = atomic_swap_int(&sc->cst_cx_lowest, val);
old_type = sc->cst_cx_states[old_lowest].type;
type = sc->cst_cx_states[val].type;
if (old_type >= ACPI_STATE_C3 && type < ACPI_STATE_C3) {
cputimer_intr_powersave_remreq();
} else if (type >= ACPI_STATE_C3 && old_type < ACPI_STATE_C3) {
error = cputimer_intr_powersave_addreq();
if (error) {
/* Restore */
sc->cst_cx_lowest_req = old_lowest_req;
sc->cst_cx_lowest = old_lowest;
}
}
if (error)
return error;
/* Cache the new lowest non-C3 state. */
acpi_cst_non_c3(sc);
/* Reset the statistics counters. */
bzero(sc->cst_cx_stats, sizeof(sc->cst_cx_stats));
return (0);
}
int
_pthread_spin_trylock(pthread_spinlock_t *lock)
{
struct pthread_spinlock *lck;
struct pthread *self = _pthread_self();
int oldval, ret;
if (lock == NULL || (lck = *lock) == NULL)
ret = EINVAL;
else if (lck->s_owner == self)
ret = EDEADLK;
else if (lck->s_lock != 0)
ret = EBUSY;
else {
atomic_swap_int(&(lck)->s_lock, 1, &oldval);
if (oldval)
ret = EBUSY;
else {
lck->s_owner = _pthread_self();
ret = 0;
}
}
return (ret);
}
/*
* Release a lock.
*/
void
_lock_release(struct lock *lck, struct lockuser *lu)
{
struct lockuser *lu_tmp, *lu_h;
struct lockreq *myreq;
int prio_h;
int lval;
/**
* XXX - We probably want to remove these checks to optimize
* performance. It is also a bug if any one of the
* checks fail, so it's probably better to just let it
* SEGV and fix it.
*/
#if 0
if ((lck == NULL) || (lu == NULL))
return;
#endif
if ((lck->l_type & LCK_PRIORITY) != 0) {
prio_h = 0;
lu_h = NULL;
/* Update tail if our request is last. */
if (lu->lu_watchreq->lr_owner == NULL) {
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lck->l_tail,
(uintptr_t)lu->lu_myreq);
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lu->lu_myreq->lr_owner,
(uintptr_t)NULL);
} else {
/* Remove ourselves from the list. */
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lu->lu_myreq->lr_owner,
(uintptr_t)lu->lu_watchreq->lr_owner);
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lu->lu_watchreq->lr_owner->lu_myreq,
(uintptr_t)lu->lu_myreq);
}
/*
* The watch request now becomes our own because we've
* traded away our previous request. Save our previous
* request so that we can grant the lock.
*/
myreq = lu->lu_myreq;
lu->lu_myreq = lu->lu_watchreq;
lu->lu_watchreq = NULL;
lu->lu_myreq->lr_locked = 1;
lu->lu_myreq->lr_owner = lu;
lu->lu_myreq->lr_watcher = NULL;
/*
* Traverse the list of lock requests in reverse order
* looking for the user with the highest priority.
*/
for (lu_tmp = lck->l_tail->lr_watcher; lu_tmp != NULL;
lu_tmp = lu_tmp->lu_myreq->lr_watcher) {
if (lu_tmp->lu_priority > prio_h) {
lu_h = lu_tmp;
prio_h = lu_tmp->lu_priority;
}
}
if (lu_h != NULL) {
/* Give the lock to the highest priority user. */
if (lck->l_wakeup != NULL) {
atomic_swap_int(
&lu_h->lu_watchreq->lr_locked,
0, &lval);
if (lval == 2)
/* Notify the sleeper */
lck->l_wakeup(lck,
lu_h->lu_myreq->lr_watcher);
}
else
atomic_store_rel_int(
&lu_h->lu_watchreq->lr_locked, 0);
} else {
if (lck->l_wakeup != NULL) {
atomic_swap_int(&myreq->lr_locked,
0, &lval);
if (lval == 2)
/* Notify the sleeper */
lck->l_wakeup(lck, myreq->lr_watcher);
}
else
/* Give the lock to the previous request. */
atomic_store_rel_int(&myreq->lr_locked, 0);
}
} else {
/*
* The watch request now becomes our own because we've
* traded away our previous request. Save our previous
* request so that we can grant the lock.
*/
myreq = lu->lu_myreq;
lu->lu_myreq = lu->lu_watchreq;
lu->lu_watchreq = NULL;
lu->lu_myreq->lr_locked = 1;
if (lck->l_wakeup) {
atomic_swap_int(&myreq->lr_locked, 0, &lval);
if (lval == 2)
/* Notify the sleeper */
lck->l_wakeup(lck, myreq->lr_watcher);
}
else
/* Give the lock to the previous request. */
atomic_store_rel_int(&myreq->lr_locked, 0);
}
lu->lu_myreq->lr_active = 0;
}
/*
* Acquire a lock waiting (spin or sleep) for it to become available.
*/
void
_lock_acquire(struct lock *lck, struct lockuser *lu, int prio)
{
int i;
int lval;
/**
* XXX - We probably want to remove these checks to optimize
* performance. It is also a bug if any one of the
* checks fail, so it's probably better to just let it
* SEGV and fix it.
*/
#if 0
if (lck == NULL || lu == NULL || lck->l_head == NULL)
return;
#endif
if ((lck->l_type & LCK_PRIORITY) != 0) {
LCK_ASSERT(lu->lu_myreq->lr_locked == 1);
LCK_ASSERT(lu->lu_myreq->lr_watcher == NULL);
LCK_ASSERT(lu->lu_myreq->lr_owner == lu);
LCK_ASSERT(lu->lu_watchreq == NULL);
lu->lu_priority = prio;
}
/*
* Atomically swap the head of the lock request with
* this request.
*/
atomic_swap_ptr((void *)&lck->l_head, lu->lu_myreq,
(void *)&lu->lu_watchreq);
if (lu->lu_watchreq->lr_locked != 0) {
atomic_store_rel_ptr
((volatile uintptr_t *)(void *)&lu->lu_watchreq->lr_watcher,
(uintptr_t)lu);
if ((lck->l_wait == NULL) ||
((lck->l_type & LCK_ADAPTIVE) == 0)) {
while (lu->lu_watchreq->lr_locked != 0)
; /* spin, then yield? */
} else {
/*
* Spin for a bit before invoking the wait function.
*
* We should be a little smarter here. If we're
* running on a single processor, then the lock
* owner got preempted and spinning will accomplish
* nothing but waste time. If we're running on
* multiple processors, the owner could be running
* on another CPU and we might acquire the lock if
* we spin for a bit.
*
* The other thing to keep in mind is that threads
* acquiring these locks are considered to be in
* critical regions; they will not be preempted by
* the _UTS_ until they release the lock. It is
* therefore safe to assume that if a lock can't
* be acquired, it is currently held by a thread
* running in another KSE.
*/
for (i = 0; i < MAX_SPINS; i++) {
if (lu->lu_watchreq->lr_locked == 0)
return;
if (lu->lu_watchreq->lr_active == 0)
break;
}
atomic_swap_int(&lu->lu_watchreq->lr_locked,
2, &lval);
if (lval == 0)
lu->lu_watchreq->lr_locked = 0;
else
lck->l_wait(lck, lu);
}
}
lu->lu_myreq->lr_active = 1;
}
static int
acpi_cpu_set_cx_lowest(struct acpi_cpu_softc *sc, int val)
{
int i, old_lowest, error = 0;
uint32_t old_type, type;
get_mplock();
old_lowest = atomic_swap_int(&sc->cpu_cx_lowest, val);
old_type = sc->cpu_cx_states[old_lowest].type;
type = sc->cpu_cx_states[val].type;
if (old_type == ACPI_STATE_C3 && type != ACPI_STATE_C3) {
KKASSERT(cpu_c3_ncpus > 0);
if (atomic_fetchadd_int(&cpu_c3_ncpus, -1) == 1) {
/*
* All of the CPUs exit C3 state, use a better
* one shot timer.
*/
error = cputimer_intr_select_caps(CPUTIMER_INTR_CAP_NONE);
KKASSERT(!error);
cputimer_intr_restart();
}
} else if (type == ACPI_STATE_C3 && old_type != ACPI_STATE_C3) {
if (atomic_fetchadd_int(&cpu_c3_ncpus, 1) == 0) {
/*
* When the first CPU enters C3 state, switch
* to an one shot timer, which could handle
* C3 state, i.e. the timer will not hang.
*/
error = cputimer_intr_select_caps(CPUTIMER_INTR_CAP_PS);
if (!error) {
cputimer_intr_restart();
} else {
kprintf("no suitable intr cputimer found\n");
/* Restore */
sc->cpu_cx_lowest = old_lowest;
atomic_fetchadd_int(&cpu_c3_ncpus, -1);
}
}
}
rel_mplock();
if (error)
return error;
/* If not disabling, cache the new lowest non-C3 state. */
sc->cpu_non_c3 = 0;
for (i = sc->cpu_cx_lowest; i >= 0; i--) {
if (sc->cpu_cx_states[i].type < ACPI_STATE_C3) {
sc->cpu_non_c3 = i;
break;
}
}
/* Reset the statistics counters. */
bzero(sc->cpu_cx_stats, sizeof(sc->cpu_cx_stats));
return (0);
}
