static void thread_uw_init(void) { static int inited = 0; Dl_info dlinfo; void *handle; void *forcedunwind, *getcfa; if (inited) return; handle = RTLD_DEFAULT; if ((forcedunwind = dlsym(handle, "_Unwind_ForcedUnwind")) != NULL) { if (dladdr(forcedunwind, &dlinfo)) { /* * Make sure the address is always valid by holding the library, * also assume functions are in same library. */ if ((handle = dlopen(dlinfo.dli_fname, RTLD_LAZY)) != NULL) { forcedunwind = dlsym(handle, "_Unwind_ForcedUnwind"); getcfa = dlsym(handle, "_Unwind_GetCFA"); if (forcedunwind != NULL && getcfa != NULL) { uwl_getcfa = getcfa; atomic_store_rel_ptr((volatile void *)&uwl_forcedunwind, (uintptr_t)forcedunwind); } else { dlclose(handle); } } } } inited = 1; }
/* * Downgrade an unrecursed exclusive lock into a single shared lock. */ void _sx_downgrade(struct sx *sx, const char *file, int line) { uintptr_t x; int wakeup_swapper; KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); #ifndef INVARIANTS if (sx_recursed(sx)) panic("downgrade of a recursed lock"); #endif WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); /* * Try to switch from an exclusive lock with no shared waiters * to one sharer with no shared waiters. If there are * exclusive waiters, we don't need to lock the sleep queue so * long as we preserve the flag. We do one quick try and if * that fails we grab the sleepq lock to keep the flags from * changing and do it the slow way. * * We have to lock the sleep queue if there are shared waiters * so we can wake them up. */ x = sx->sx_lock; if (!(x & SX_LOCK_SHARED_WAITERS) && atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | (x & SX_LOCK_EXCLUSIVE_WAITERS))) { LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); return; } /* * Lock the sleep queue so we can read the waiters bits * without any races and wakeup any shared waiters. */ sleepq_lock(&sx->lock_object); /* * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single * shared lock. If there are any shared waiters, wake them up. */ wakeup_swapper = 0; x = sx->sx_lock; atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | (x & SX_LOCK_EXCLUSIVE_WAITERS)); if (x & SX_LOCK_SHARED_WAITERS) wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, SQ_SHARED_QUEUE); sleepq_release(&sx->lock_object); LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx); if (wakeup_swapper) kick_proc0(); }
static int ismt_callback(device_t dev, int index, void *data) { struct ismt_softc *sc; int acquired, err; sc = device_get_softc(dev); switch (index) { case SMB_REQUEST_BUS: acquired = atomic_cmpset_ptr( (uintptr_t *)&sc->bus_reserved, (uintptr_t)NULL, (uintptr_t)curthread); ISMT_DEBUG(dev, "SMB_REQUEST_BUS acquired=%d\n", acquired); if (acquired) err = 0; else err = EWOULDBLOCK; break; case SMB_RELEASE_BUS: KASSERT(sc->bus_reserved == curthread, ("SMB_RELEASE_BUS called by wrong thread\n")); ISMT_DEBUG(dev, "SMB_RELEASE_BUS\n"); atomic_store_rel_ptr((uintptr_t *)&sc->bus_reserved, (uintptr_t)NULL); err = 0; break; default: err = SMB_EABORT; break; } return (err); }
/* * This function is called if the first try at releasing a write lock failed. * This means that one of the 2 waiter bits must be set indicating that at * least one thread is waiting on this lock. */ void _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) { struct turnstile *ts; uintptr_t v; int queue; if (SCHEDULER_STOPPED()) return; if (rw_wlocked(rw) && rw_recursed(rw)) { rw->rw_recurse--; if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); return; } KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); turnstile_chain_lock(&rw->lock_object); ts = turnstile_lookup(&rw->lock_object); MPASS(ts != NULL); /* * Use the same algo as sx locks for now. Prefer waking up shared * waiters if we have any over writers. This is probably not ideal. * * 'v' is the value we are going to write back to rw_lock. If we * have waiters on both queues, we need to preserve the state of * the waiter flag for the queue we don't wake up. For now this is * hardcoded for the algorithm mentioned above. * * In the case of both readers and writers waiting we wakeup the * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a * new writer comes in before a reader it will claim the lock up * above. There is probably a potential priority inversion in * there that could be worked around either by waking both queues * of waiters or doing some complicated lock handoff gymnastics. */ v = RW_UNLOCKED; if (rw->rw_lock & RW_LOCK_WRITE_WAITERS) { queue = TS_EXCLUSIVE_QUEUE; v |= (rw->rw_lock & RW_LOCK_READ_WAITERS); } else queue = TS_SHARED_QUEUE; /* Wake up all waiters for the specific queue. */ if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, queue == TS_SHARED_QUEUE ? "read" : "write"); turnstile_broadcast(ts, queue); atomic_store_rel_ptr(&rw->rw_lock, v); turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); turnstile_chain_unlock(&rw->lock_object); }
/* * This function represents the so-called 'hard case' for sx_xunlock * operation. All 'easy case' failures are redirected to this. Note * that ideally this would be a static function, but it needs to be * accessible from at least sx.h. */ void _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line) { uintptr_t x; int queue, wakeup_swapper; if (SCHEDULER_STOPPED()) return; MPASS(!(sx->sx_lock & SX_LOCK_SHARED)); /* If the lock is recursed, then unrecurse one level. */ if (sx_xlocked(sx) && sx_recursed(sx)) { if ((--sx->sx_recurse) == 0) atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED); if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx); return; } MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)); if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p contested", __func__, sx); sleepq_lock(&sx->lock_object); x = SX_LOCK_UNLOCKED; /* * The wake up algorithm here is quite simple and probably not * ideal. It gives precedence to shared waiters if they are * present. For this condition, we have to preserve the * state of the exclusive waiters flag. * If interruptible sleeps left the shared queue empty avoid a * starvation for the threads sleeping on the exclusive queue by giving * them precedence and cleaning up the shared waiters bit anyway. */ if ((sx->sx_lock & SX_LOCK_SHARED_WAITERS) != 0 && sleepq_sleepcnt(&sx->lock_object, SQ_SHARED_QUEUE) != 0) { queue = SQ_SHARED_QUEUE; x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS); } else queue = SQ_EXCLUSIVE_QUEUE; /* Wake up all the waiters for the specific queue. */ if (LOCK_LOG_TEST(&sx->lock_object, 0)) CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue", __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" : "exclusive"); atomic_store_rel_ptr(&sx->sx_lock, x); wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, queue); sleepq_release(&sx->lock_object); if (wakeup_swapper) kick_proc0(); }
static int bhndb_pci_attach(device_t dev) { struct bhndb_pci_softc *sc; int error, reg; sc = device_get_softc(dev); sc->dev = dev; /* Enable PCI bus mastering */ pci_enable_busmaster(device_get_parent(dev)); /* Determine our bridge device class */ sc->pci_devclass = BHND_DEVCLASS_PCI; if (pci_find_cap(device_get_parent(dev), PCIY_EXPRESS, ®) == 0) sc->pci_devclass = BHND_DEVCLASS_PCIE; /* Determine the basic set of applicable quirks. This will be updated * in bhndb_pci_init_full_config() once the PCI device core has * been enumerated. */ sc->quirks = bhndb_pci_discover_quirks(sc, NULL); /* Using the discovered quirks, apply any WARs required for basic * register access. */ if ((error = bhndb_pci_wars_register_access(sc))) return (error); /* Use siba(4)-compatible regwin handling until we know * what kind of bus is attached */ sc->set_regwin = bhndb_pci_compat_setregwin; /* Perform full bridge attach. This should call back into our * bhndb_pci_init_full_config() implementation once the bridged * bhnd(4) bus has been enumerated, but before any devices have been * probed or attached. */ if ((error = bhndb_attach(dev, sc->pci_devclass))) return (error); /* If supported, switch to the faster regwin handling */ if (sc->bhndb.chipid.chip_type != BHND_CHIPTYPE_SIBA) { atomic_store_rel_ptr((volatile void *) &sc->set_regwin, (uintptr_t) &bhndb_pci_fast_setregwin); } return (0); }
/* * Downgrade a write lock into a single read lock. */ void __rw_downgrade(volatile uintptr_t *c, const char *file, int line) { struct rwlock *rw; struct turnstile *ts; uintptr_t tid, v; int rwait, wwait; if (SCHEDULER_STOPPED()) return; rw = rwlock2rw(c); KASSERT(rw->rw_lock != RW_DESTROYED, ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line)); __rw_assert(c, RA_WLOCKED | RA_NOTRECURSED, file, line); #ifndef INVARIANTS if (rw_recursed(rw)) panic("downgrade of a recursed lock"); #endif WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line); /* * Convert from a writer to a single reader. First we handle * the easy case with no waiters. If there are any waiters, we * lock the turnstile and "disown" the lock. */ tid = (uintptr_t)curthread; if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1))) goto out; /* * Ok, we think we have waiters, so lock the turnstile so we can * read the waiter flags without any races. */ turnstile_chain_lock(&rw->lock_object); v = rw->rw_lock & RW_LOCK_WAITERS; rwait = v & RW_LOCK_READ_WAITERS; wwait = v & RW_LOCK_WRITE_WAITERS; MPASS(rwait | wwait); /* * Downgrade from a write lock while preserving waiters flag * and give up ownership of the turnstile. */ ts = turnstile_lookup(&rw->lock_object); MPASS(ts != NULL); if (!wwait) v &= ~RW_LOCK_READ_WAITERS; atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v); /* * Wake other readers if there are no writers pending. Otherwise they * won't be able to acquire the lock anyway. */ if (rwait && !wwait) { turnstile_broadcast(ts, TS_SHARED_QUEUE); turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); } else turnstile_disown(ts); turnstile_chain_unlock(&rw->lock_object); out: curthread->td_rw_rlocks++; LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line); LOCKSTAT_RECORD0(rw__downgrade, rw); }
/* * Release a lock. */ void _lock_release(struct lock *lck, struct lockuser *lu) { struct lockuser *lu_tmp, *lu_h; struct lockreq *myreq; int prio_h; int lval; /** * XXX - We probably want to remove these checks to optimize * performance. It is also a bug if any one of the * checks fail, so it's probably better to just let it * SEGV and fix it. */ #if 0 if ((lck == NULL) || (lu == NULL)) return; #endif if ((lck->l_type & LCK_PRIORITY) != 0) { prio_h = 0; lu_h = NULL; /* Update tail if our request is last. */ if (lu->lu_watchreq->lr_owner == NULL) { atomic_store_rel_ptr((volatile uintptr_t *) (void *)&lck->l_tail, (uintptr_t)lu->lu_myreq); atomic_store_rel_ptr((volatile uintptr_t *) (void *)&lu->lu_myreq->lr_owner, (uintptr_t)NULL); } else { /* Remove ourselves from the list. */ atomic_store_rel_ptr((volatile uintptr_t *) (void *)&lu->lu_myreq->lr_owner, (uintptr_t)lu->lu_watchreq->lr_owner); atomic_store_rel_ptr((volatile uintptr_t *) (void *)&lu->lu_watchreq->lr_owner->lu_myreq, (uintptr_t)lu->lu_myreq); } /* * The watch request now becomes our own because we've * traded away our previous request. Save our previous * request so that we can grant the lock. */ myreq = lu->lu_myreq; lu->lu_myreq = lu->lu_watchreq; lu->lu_watchreq = NULL; lu->lu_myreq->lr_locked = 1; lu->lu_myreq->lr_owner = lu; lu->lu_myreq->lr_watcher = NULL; /* * Traverse the list of lock requests in reverse order * looking for the user with the highest priority. */ for (lu_tmp = lck->l_tail->lr_watcher; lu_tmp != NULL; lu_tmp = lu_tmp->lu_myreq->lr_watcher) { if (lu_tmp->lu_priority > prio_h) { lu_h = lu_tmp; prio_h = lu_tmp->lu_priority; } } if (lu_h != NULL) { /* Give the lock to the highest priority user. */ if (lck->l_wakeup != NULL) { atomic_swap_int( &lu_h->lu_watchreq->lr_locked, 0, &lval); if (lval == 2) /* Notify the sleeper */ lck->l_wakeup(lck, lu_h->lu_myreq->lr_watcher); } else atomic_store_rel_int( &lu_h->lu_watchreq->lr_locked, 0); } else { if (lck->l_wakeup != NULL) { atomic_swap_int(&myreq->lr_locked, 0, &lval); if (lval == 2) /* Notify the sleeper */ lck->l_wakeup(lck, myreq->lr_watcher); } else /* Give the lock to the previous request. */ atomic_store_rel_int(&myreq->lr_locked, 0); } } else { /* * The watch request now becomes our own because we've * traded away our previous request. Save our previous * request so that we can grant the lock. */ myreq = lu->lu_myreq; lu->lu_myreq = lu->lu_watchreq; lu->lu_watchreq = NULL; lu->lu_myreq->lr_locked = 1; if (lck->l_wakeup) { atomic_swap_int(&myreq->lr_locked, 0, &lval); if (lval == 2) /* Notify the sleeper */ lck->l_wakeup(lck, myreq->lr_watcher); } else /* Give the lock to the previous request. */ atomic_store_rel_int(&myreq->lr_locked, 0); } lu->lu_myreq->lr_active = 0; }
/* * Acquire a lock waiting (spin or sleep) for it to become available. */ void _lock_acquire(struct lock *lck, struct lockuser *lu, int prio) { int i; int lval; /** * XXX - We probably want to remove these checks to optimize * performance. It is also a bug if any one of the * checks fail, so it's probably better to just let it * SEGV and fix it. */ #if 0 if (lck == NULL || lu == NULL || lck->l_head == NULL) return; #endif if ((lck->l_type & LCK_PRIORITY) != 0) { LCK_ASSERT(lu->lu_myreq->lr_locked == 1); LCK_ASSERT(lu->lu_myreq->lr_watcher == NULL); LCK_ASSERT(lu->lu_myreq->lr_owner == lu); LCK_ASSERT(lu->lu_watchreq == NULL); lu->lu_priority = prio; } /* * Atomically swap the head of the lock request with * this request. */ atomic_swap_ptr((void *)&lck->l_head, lu->lu_myreq, (void *)&lu->lu_watchreq); if (lu->lu_watchreq->lr_locked != 0) { atomic_store_rel_ptr ((volatile uintptr_t *)(void *)&lu->lu_watchreq->lr_watcher, (uintptr_t)lu); if ((lck->l_wait == NULL) || ((lck->l_type & LCK_ADAPTIVE) == 0)) { while (lu->lu_watchreq->lr_locked != 0) ; /* spin, then yield? */ } else { /* * Spin for a bit before invoking the wait function. * * We should be a little smarter here. If we're * running on a single processor, then the lock * owner got preempted and spinning will accomplish * nothing but waste time. If we're running on * multiple processors, the owner could be running * on another CPU and we might acquire the lock if * we spin for a bit. * * The other thing to keep in mind is that threads * acquiring these locks are considered to be in * critical regions; they will not be preempted by * the _UTS_ until they release the lock. It is * therefore safe to assume that if a lock can't * be acquired, it is currently held by a thread * running in another KSE. */ for (i = 0; i < MAX_SPINS; i++) { if (lu->lu_watchreq->lr_locked == 0) return; if (lu->lu_watchreq->lr_active == 0) break; } atomic_swap_int(&lu->lu_watchreq->lr_locked, 2, &lval); if (lval == 0) lu->lu_watchreq->lr_locked = 0; else lck->l_wait(lck, lu); } } lu->lu_myreq->lr_active = 1; }
t_Handle bman_portal_setup(struct bman_softc *bsc) { struct dpaa_portals_softc *sc; t_BmPortalParam bpp; t_Handle portal; unsigned int cpu; uintptr_t p; /* Return NULL if we're not ready or while detach */ if (bp_sc == NULL) return (NULL); sc = bp_sc; sched_pin(); portal = NULL; cpu = PCPU_GET(cpuid); /* Check if portal is ready */ while (atomic_cmpset_acq_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph, 0, -1) == 0) { p = atomic_load_acq_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph); /* Return if portal is already initialized */ if (p != 0 && p != -1) { sched_unpin(); return ((t_Handle)p); } /* Not inititialized and "owned" by another thread */ thread_lock(curthread); mi_switch(SW_VOL, NULL); thread_unlock(curthread); } /* Map portal registers */ dpaa_portal_map_registers(sc); /* Configure and initialize portal */ bpp.ceBaseAddress = rman_get_bushandle(sc->sc_rres[0]); bpp.ciBaseAddress = rman_get_bushandle(sc->sc_rres[1]); bpp.h_Bm = bsc->sc_bh; bpp.swPortalId = cpu; bpp.irq = (uintptr_t)sc->sc_dp[cpu].dp_ires; portal = BM_PORTAL_Config(&bpp); if (portal == NULL) goto err; if (BM_PORTAL_Init(portal) != E_OK) goto err; atomic_store_rel_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph, (uintptr_t)portal); sched_unpin(); return (portal); err: if (portal != NULL) BM_PORTAL_Free(portal); atomic_store_rel_ptr((uintptr_t *)&sc->sc_dp[cpu].dp_ph, 0); sched_unpin(); return (NULL); }