static void
thread_uw_init(void)
{
static int inited = 0;
Dl_info dlinfo;
void *handle;
void *forcedunwind, *getcfa;
if (inited)
return;
handle = RTLD_DEFAULT;
if ((forcedunwind = dlsym(handle, "_Unwind_ForcedUnwind")) != NULL) {
if (dladdr(forcedunwind, &dlinfo)) {
/*
* Make sure the address is always valid by holding the library,
* also assume functions are in same library.
*/
if ((handle = dlopen(dlinfo.dli_fname, RTLD_LAZY)) != NULL) {
forcedunwind = dlsym(handle, "_Unwind_ForcedUnwind");
getcfa = dlsym(handle, "_Unwind_GetCFA");
if (forcedunwind != NULL && getcfa != NULL) {
uwl_getcfa = getcfa;
atomic_store_rel_ptr((volatile void *)&uwl_forcedunwind,
(uintptr_t)forcedunwind);
} else {
dlclose(handle);
}
}
}
}
inited = 1;
}
/*
* Downgrade an unrecursed exclusive lock into a single shared lock.
*/
void
_sx_downgrade(struct sx *sx, const char *file, int line)
{
uintptr_t x;
int wakeup_swapper;
KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
("sx_downgrade() of destroyed sx @ %s:%d", file, line));
_sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
#ifndef INVARIANTS
if (sx_recursed(sx))
panic("downgrade of a recursed lock");
#endif
WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
/*
* Try to switch from an exclusive lock with no shared waiters
* to one sharer with no shared waiters. If there are
* exclusive waiters, we don't need to lock the sleep queue so
* long as we preserve the flag. We do one quick try and if
* that fails we grab the sleepq lock to keep the flags from
* changing and do it the slow way.
*
* We have to lock the sleep queue if there are shared waiters
* so we can wake them up.
*/
x = sx->sx_lock;
if (!(x & SX_LOCK_SHARED_WAITERS) &&
atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) |
(x & SX_LOCK_EXCLUSIVE_WAITERS))) {
LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
return;
}
/*
* Lock the sleep queue so we can read the waiters bits
* without any races and wakeup any shared waiters.
*/
sleepq_lock(&sx->lock_object);
/*
* Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
* shared lock. If there are any shared waiters, wake them up.
*/
wakeup_swapper = 0;
x = sx->sx_lock;
atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
(x & SX_LOCK_EXCLUSIVE_WAITERS));
if (x & SX_LOCK_SHARED_WAITERS)
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX,
0, SQ_SHARED_QUEUE);
sleepq_release(&sx->lock_object);
LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx);
if (wakeup_swapper)
kick_proc0();
}
static int
ismt_callback(device_t dev, int index, void *data)
{
struct ismt_softc *sc;
int acquired, err;
sc = device_get_softc(dev);
switch (index) {
case SMB_REQUEST_BUS:
acquired = atomic_cmpset_ptr(
(uintptr_t *)&sc->bus_reserved,
(uintptr_t)NULL, (uintptr_t)curthread);
ISMT_DEBUG(dev, "SMB_REQUEST_BUS acquired=%d\n", acquired);
if (acquired)
err = 0;
else
err = EWOULDBLOCK;
break;
case SMB_RELEASE_BUS:
KASSERT(sc->bus_reserved == curthread,
("SMB_RELEASE_BUS called by wrong thread\n"));
ISMT_DEBUG(dev, "SMB_RELEASE_BUS\n");
atomic_store_rel_ptr((uintptr_t *)&sc->bus_reserved,
(uintptr_t)NULL);
err = 0;
break;
default:
err = SMB_EABORT;
break;
}
return (err);
}
/*
* This function is called if the first try at releasing a write lock failed.
* This means that one of the 2 waiter bits must be set indicating that at
* least one thread is waiting on this lock.
*/
void
_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
{
struct turnstile *ts;
uintptr_t v;
int queue;
if (SCHEDULER_STOPPED())
return;
if (rw_wlocked(rw) && rw_recursed(rw)) {
rw->rw_recurse--;
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
return;
}
KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
("%s: neither of the waiter flags are set", __func__));
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
turnstile_chain_lock(&rw->lock_object);
ts = turnstile_lookup(&rw->lock_object);
MPASS(ts != NULL);
/*
* Use the same algo as sx locks for now. Prefer waking up shared
* waiters if we have any over writers. This is probably not ideal.
*
* 'v' is the value we are going to write back to rw_lock. If we
* have waiters on both queues, we need to preserve the state of
* the waiter flag for the queue we don't wake up. For now this is
* hardcoded for the algorithm mentioned above.
*
* In the case of both readers and writers waiting we wakeup the
* readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a
* new writer comes in before a reader it will claim the lock up
* above. There is probably a potential priority inversion in
* there that could be worked around either by waking both queues
* of waiters or doing some complicated lock handoff gymnastics.
*/
v = RW_UNLOCKED;
if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) {
queue = TS_EXCLUSIVE_QUEUE;
v |= (rw->rw_lock & RW_LOCK_READ_WAITERS);
} else
queue = TS_SHARED_QUEUE;
/* Wake up all waiters for the specific queue. */
if (LOCK_LOG_TEST(&rw->lock_object, 0))
CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
queue == TS_SHARED_QUEUE ? "read" : "write");
turnstile_broadcast(ts, queue);
atomic_store_rel_ptr(&rw->rw_lock, v);
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
turnstile_chain_unlock(&rw->lock_object);
}
/*
* This function represents the so-called 'hard case' for sx_xunlock
* operation. All 'easy case' failures are redirected to this. Note
* that ideally this would be a static function, but it needs to be
* accessible from at least sx.h.
*/
void
_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
{
uintptr_t x;
int queue, wakeup_swapper;
if (SCHEDULER_STOPPED())
return;
MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
/* If the lock is recursed, then unrecurse one level. */
if (sx_xlocked(sx) && sx_recursed(sx)) {
if ((--sx->sx_recurse) == 0)
atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
return;
}
MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS |
SX_LOCK_EXCLUSIVE_WAITERS));
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR2(KTR_LOCK, "%s: %p contested", __func__, sx);
sleepq_lock(&sx->lock_object);
x = SX_LOCK_UNLOCKED;
/*
* The wake up algorithm here is quite simple and probably not
* ideal. It gives precedence to shared waiters if they are
* present. For this condition, we have to preserve the
* state of the exclusive waiters flag.
* If interruptible sleeps left the shared queue empty avoid a
* starvation for the threads sleeping on the exclusive queue by giving
* them precedence and cleaning up the shared waiters bit anyway.
*/
if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 &&
sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) {
queue = SQ_SHARED_QUEUE;
x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS);
} else
queue = SQ_EXCLUSIVE_QUEUE;
/* Wake up all the waiters for the specific queue. */
if (LOCK_LOG_TEST(&sx->lock_object, 0))
CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
__func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
"exclusive");
atomic_store_rel_ptr(&sx->sx_lock, x);
wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0,
queue);
sleepq_release(&sx->lock_object);
if (wakeup_swapper)
kick_proc0();
}
static int
bhndb_pci_attach(device_t dev)
{
struct bhndb_pci_softc *sc;
int error, reg;
sc = device_get_softc(dev);
sc->dev = dev;
/* Enable PCI bus mastering */
pci_enable_busmaster(device_get_parent(dev));
/* Determine our bridge device class */
sc->pci_devclass = BHND_DEVCLASS_PCI;
if (pci_find_cap(device_get_parent(dev), PCIY_EXPRESS, ®) == 0)
sc->pci_devclass = BHND_DEVCLASS_PCIE;
/* Determine the basic set of applicable quirks. This will be updated
* in bhndb_pci_init_full_config() once the PCI device core has
* been enumerated. */
sc->quirks = bhndb_pci_discover_quirks(sc, NULL);
/* Using the discovered quirks, apply any WARs required for basic
* register access. */
if ((error = bhndb_pci_wars_register_access(sc)))
return (error);
/* Use siba(4)-compatible regwin handling until we know
* what kind of bus is attached */
sc->set_regwin = bhndb_pci_compat_setregwin;
/* Perform full bridge attach. This should call back into our
* bhndb_pci_init_full_config() implementation once the bridged
* bhnd(4) bus has been enumerated, but before any devices have been
* probed or attached. */
if ((error = bhndb_attach(dev, sc->pci_devclass)))
return (error);
/* If supported, switch to the faster regwin handling */
if (sc->bhndb.chipid.chip_type != BHND_CHIPTYPE_SIBA) {
atomic_store_rel_ptr((volatile void *) &sc->set_regwin,
(uintptr_t) &bhndb_pci_fast_setregwin);
}
return (0);
}
/*
* Downgrade a write lock into a single read lock.
*/
void
__rw_downgrade(volatile uintptr_t *c, const char *file, int line)
{
struct rwlock *rw;
struct turnstile *ts;
uintptr_t tid, v;
int rwait, wwait;
if (SCHEDULER_STOPPED())
return;
rw = rwlock2rw(c);
KASSERT(rw->rw_lock != RW_DESTROYED,
("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
__rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line);
#ifndef INVARIANTS
if (rw_recursed(rw))
panic("downgrade of a recursed lock");
#endif
WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
/*
* Convert from a writer to a single reader. First we handle
* the easy case with no waiters. If there are any waiters, we
* lock the turnstile and "disown" the lock.
*/
tid = (uintptr_t)curthread;
if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
goto out;
/*
* Ok, we think we have waiters, so lock the turnstile so we can
* read the waiter flags without any races.
*/
turnstile_chain_lock(&rw->lock_object);
v = rw->rw_lock & RW_LOCK_WAITERS;
rwait = v & RW_LOCK_READ_WAITERS;
wwait = v & RW_LOCK_WRITE_WAITERS;
MPASS(rwait | wwait);
/*
* Downgrade from a write lock while preserving waiters flag
* and give up ownership of the turnstile.
*/
ts = turnstile_lookup(&rw->lock_object);
MPASS(ts != NULL);
if (!wwait)
v &= ~RW_LOCK_READ_WAITERS;
atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v);
/*
* Wake other readers if there are no writers pending. Otherwise they
* won't be able to acquire the lock anyway.
*/
if (rwait && !wwait) {
turnstile_broadcast(ts, TS_SHARED_QUEUE);
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
} else
turnstile_disown(ts);
turnstile_chain_unlock(&rw->lock_object);
out:
curthread->td_rw_rlocks++;
LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
LOCKSTAT_RECORD0(rw__downgrade, rw);
}
/*
* Release a lock.
*/
void
_lock_release(struct lock *lck, struct lockuser *lu)
{
struct lockuser *lu_tmp, *lu_h;
struct lockreq *myreq;
int prio_h;
int lval;
/**
* XXX - We probably want to remove these checks to optimize
* performance. It is also a bug if any one of the
* checks fail, so it's probably better to just let it
* SEGV and fix it.
*/
#if 0
if ((lck == NULL) || (lu == NULL))
return;
#endif
if ((lck->l_type & LCK_PRIORITY) != 0) {
prio_h = 0;
lu_h = NULL;
/* Update tail if our request is last. */
if (lu->lu_watchreq->lr_owner == NULL) {
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lck->l_tail,
(uintptr_t)lu->lu_myreq);
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lu->lu_myreq->lr_owner,
(uintptr_t)NULL);
} else {
/* Remove ourselves from the list. */
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lu->lu_myreq->lr_owner,
(uintptr_t)lu->lu_watchreq->lr_owner);
atomic_store_rel_ptr((volatile uintptr_t *)
(void *)&lu->lu_watchreq->lr_owner->lu_myreq,
(uintptr_t)lu->lu_myreq);
}
/*
* The watch request now becomes our own because we've
* traded away our previous request. Save our previous
* request so that we can grant the lock.
*/
myreq = lu->lu_myreq;
lu->lu_myreq = lu->lu_watchreq;
lu->lu_watchreq = NULL;
lu->lu_myreq->lr_locked = 1;
lu->lu_myreq->lr_owner = lu;
lu->lu_myreq->lr_watcher = NULL;
/*
* Traverse the list of lock requests in reverse order
* looking for the user with the highest priority.
*/
for (lu_tmp = lck->l_tail->lr_watcher; lu_tmp != NULL;
lu_tmp = lu_tmp->lu_myreq->lr_watcher) {
if (lu_tmp->lu_priority > prio_h) {
lu_h = lu_tmp;
prio_h = lu_tmp->lu_priority;
}
}
if (lu_h != NULL) {
/* Give the lock to the highest priority user. */
if (lck->l_wakeup != NULL) {
atomic_swap_int(
&lu_h->lu_watchreq->lr_locked,
0, &lval);
if (lval == 2)
/* Notify the sleeper */
lck->l_wakeup(lck,
lu_h->lu_myreq->lr_watcher);
}
else
atomic_store_rel_int(
&lu_h->lu_watchreq->lr_locked, 0);
} else {
if (lck->l_wakeup != NULL) {
atomic_swap_int(&myreq->lr_locked,
0, &lval);
if (lval == 2)
/* Notify the sleeper */
lck->l_wakeup(lck, myreq->lr_watcher);
}
else
/* Give the lock to the previous request. */
atomic_store_rel_int(&myreq->lr_locked, 0);
}
} else {
/*
* The watch request now becomes our own because we've
* traded away our previous request. Save our previous
* request so that we can grant the lock.
*/
myreq = lu->lu_myreq;
lu->lu_myreq = lu->lu_watchreq;
lu->lu_watchreq = NULL;
lu->lu_myreq->lr_locked = 1;
if (lck->l_wakeup) {
atomic_swap_int(&myreq->lr_locked, 0, &lval);
if (lval == 2)
/* Notify the sleeper */
lck->l_wakeup(lck, myreq->lr_watcher);
}
else
/* Give the lock to the previous request. */
atomic_store_rel_int(&myreq->lr_locked, 0);
}
lu->lu_myreq->lr_active = 0;
}
/*
* Acquire a lock waiting (spin or sleep) for it to become available.
*/
void
_lock_acquire(struct lock *lck, struct lockuser *lu, int prio)
{
int i;
int lval;
/**
* XXX - We probably want to remove these checks to optimize
* performance. It is also a bug if any one of the
* checks fail, so it's probably better to just let it
* SEGV and fix it.
*/
#if 0
if (lck == NULL || lu == NULL || lck->l_head == NULL)
return;
#endif
if ((lck->l_type & LCK_PRIORITY) != 0) {
LCK_ASSERT(lu->lu_myreq->lr_locked == 1);
LCK_ASSERT(lu->lu_myreq->lr_watcher == NULL);
LCK_ASSERT(lu->lu_myreq->lr_owner == lu);
LCK_ASSERT(lu->lu_watchreq == NULL);
lu->lu_priority = prio;
}
/*
* Atomically swap the head of the lock request with
* this request.
*/
atomic_swap_ptr((void *)&lck->l_head, lu->lu_myreq,
(void *)&lu->lu_watchreq);
if (lu->lu_watchreq->lr_locked != 0) {
atomic_store_rel_ptr
((volatile uintptr_t *)(void *)&lu->lu_watchreq->lr_watcher,
(uintptr_t)lu);
if ((lck->l_wait == NULL) ||
((lck->l_type & LCK_ADAPTIVE) == 0)) {
while (lu->lu_watchreq->lr_locked != 0)
; /* spin, then yield? */
} else {
/*
* Spin for a bit before invoking the wait function.
*
* We should be a little smarter here. If we're
* running on a single processor, then the lock
* owner got preempted and spinning will accomplish
* nothing but waste time. If we're running on
* multiple processors, the owner could be running
* on another CPU and we might acquire the lock if
* we spin for a bit.
*
* The other thing to keep in mind is that threads
* acquiring these locks are considered to be in
* critical regions; they will not be preempted by
* the _UTS_ until they release the lock. It is
* therefore safe to assume that if a lock can't
* be acquired, it is currently held by a thread
* running in another KSE.
*/
for (i = 0; i < MAX_SPINS; i++) {
if (lu->lu_watchreq->lr_locked == 0)
return;
if (lu->lu_watchreq->lr_active == 0)
break;
}
atomic_swap_int(&lu->lu_watchreq->lr_locked,
2, &lval);
if (lval == 0)
lu->lu_watchreq->lr_locked = 0;
else
lck->l_wait(lck, lu);
}
}
lu->lu_myreq->lr_active = 1;
}
t_Handle
bman_portal_setup(struct bman_softc *bsc)
{
struct dpaa_portals_softc *sc;
t_BmPortalParam bpp;
t_Handle portal;
unsigned int cpu;
uintptr_t p;
/* Return NULL if we're not ready or while detach */
if (bp_sc == NULL)
return (NULL);
sc = bp_sc;
sched_pin();
portal = NULL;
cpu = PCPU_GET(cpuid);
/* Check if portal is ready */
while (atomic_cmpset_acq_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph,
0, -1) == 0) {
p = atomic_load_acq_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph);
/* Return if portal is already initialized */
if (p != 0 && p != -1) {
sched_unpin();
return ((t_Handle)p);
}
/* Not inititialized and "owned" by another thread */
thread_lock(curthread);
mi_switch(SW_VOL, NULL);
thread_unlock(curthread);
}
/* Map portal registers */
dpaa_portal_map_registers(sc);
/* Configure and initialize portal */
bpp.ceBaseAddress = rman_get_bushandle(sc->sc_rres[0]);
bpp.ciBaseAddress = rman_get_bushandle(sc->sc_rres[1]);
bpp.h_Bm = bsc->sc_bh;
bpp.swPortalId = cpu;
bpp.irq = (uintptr_t)sc->sc_dp[cpu].dp_ires;
portal = BM_PORTAL_Config(&bpp);
if (portal == NULL)
goto err;
if (BM_PORTAL_Init(portal) != E_OK)
goto err;
atomic_store_rel_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph, (uintptr_t)portal);
sched_unpin();
return (portal);
err:
if (portal != NULL)
BM_PORTAL_Free(portal);
atomic_store_rel_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph, 0);
sched_unpin();
return (NULL);
}
