/* * mutex_vector_enter: * * Support routine for mutex_enter() that must handle all cases. In * the LOCKDEBUG case, mutex_enter() is always aliased here, even if * fast-path stubs are available. If a mutex_spin_enter() stub is * not available, then it is also aliased directly here. */ void mutex_vector_enter(kmutex_t *mtx) { uintptr_t owner, curthread; turnstile_t *ts; #ifdef MULTIPROCESSOR u_int count; #endif LOCKSTAT_COUNTER(spincnt); LOCKSTAT_COUNTER(slpcnt); LOCKSTAT_TIMER(spintime); LOCKSTAT_TIMER(slptime); LOCKSTAT_FLAG(lsflag); /* * Handle spin mutexes. */ if (MUTEX_SPIN_P(mtx)) { #if defined(LOCKDEBUG) && defined(MULTIPROCESSOR) u_int spins = 0; #endif MUTEX_SPIN_SPLRAISE(mtx); MUTEX_WANTLOCK(mtx); #ifdef FULL if (MUTEX_SPINBIT_LOCK_TRY(mtx)) { MUTEX_LOCKED(mtx); return; } #if !defined(MULTIPROCESSOR) MUTEX_ABORT(mtx, "locking against myself"); #else /* !MULTIPROCESSOR */ LOCKSTAT_ENTER(lsflag); LOCKSTAT_START_TIMER(lsflag, spintime); count = SPINLOCK_BACKOFF_MIN; /* * Spin testing the lock word and do exponential backoff * to reduce cache line ping-ponging between CPUs. */ do { if (panicstr != NULL) break; while (MUTEX_SPINBIT_LOCKED_P(mtx)) { SPINLOCK_BACKOFF(count); #ifdef LOCKDEBUG if (SPINLOCK_SPINOUT(spins)) MUTEX_ABORT(mtx, "spinout"); #endif /* LOCKDEBUG */ } } while (!MUTEX_SPINBIT_LOCK_TRY(mtx)); if (count != SPINLOCK_BACKOFF_MIN) { LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime); } LOCKSTAT_EXIT(lsflag); #endif /* !MULTIPROCESSOR */ #endif /* FULL */ MUTEX_LOCKED(mtx); return; } curthread = (uintptr_t)curlwp; MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); MUTEX_ASSERT(mtx, curthread != 0); MUTEX_WANTLOCK(mtx); if (panicstr == NULL) { LOCKDEBUG_BARRIER(&kernel_lock, 1); } LOCKSTAT_ENTER(lsflag); /* * Adaptive mutex; spin trying to acquire the mutex. If we * determine that the owner is not running on a processor, * then we stop spinning, and sleep instead. */ KPREEMPT_DISABLE(curlwp); for (owner = mtx->mtx_owner;;) { if (!MUTEX_OWNED(owner)) { /* * Mutex owner clear could mean two things: * * * The mutex has been released. * * The owner field hasn't been set yet. * * Try to acquire it again. If that fails, * we'll just loop again. */ if (MUTEX_ACQUIRE(mtx, curthread)) break; owner = mtx->mtx_owner; continue; } if (__predict_false(panicstr != NULL)) { KPREEMPT_ENABLE(curlwp); return; } if (__predict_false(MUTEX_OWNER(owner) == curthread)) { MUTEX_ABORT(mtx, "locking against myself"); } #ifdef MULTIPROCESSOR /* * Check to see if the owner is running on a processor. * If so, then we should just spin, as the owner will * likely release the lock very soon. */ if (mutex_oncpu(owner)) { LOCKSTAT_START_TIMER(lsflag, spintime); count = SPINLOCK_BACKOFF_MIN; do { KPREEMPT_ENABLE(curlwp); SPINLOCK_BACKOFF(count); KPREEMPT_DISABLE(curlwp); owner = mtx->mtx_owner; } while (mutex_oncpu(owner)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if (!MUTEX_OWNED(owner)) continue; } #endif ts = turnstile_lookup(mtx); /* * Once we have the turnstile chain interlock, mark the * mutex has having waiters. If that fails, spin again: * chances are that the mutex has been released. */ if (!MUTEX_SET_WAITERS(mtx, owner)) { turnstile_exit(mtx); owner = mtx->mtx_owner; continue; } #ifdef MULTIPROCESSOR /* * mutex_exit() is permitted to release the mutex without * any interlocking instructions, and the following can * occur as a result: * * CPU 1: MUTEX_SET_WAITERS() CPU2: mutex_exit() * ---------------------------- ---------------------------- * .. acquire cache line * .. test for waiters * acquire cache line <- lose cache line * lock cache line .. * verify mutex is held .. * set waiters .. * unlock cache line .. * lose cache line -> acquire cache line * .. clear lock word, waiters * return success * * There is another race that can occur: a third CPU could * acquire the mutex as soon as it is released. Since * adaptive mutexes are primarily spin mutexes, this is not * something that we need to worry about too much. What we * do need to ensure is that the waiters bit gets set. * * To allow the unlocked release, we need to make some * assumptions here: * * o Release is the only non-atomic/unlocked operation * that can be performed on the mutex. (It must still * be atomic on the local CPU, e.g. in case interrupted * or preempted). * * o At any given time, MUTEX_SET_WAITERS() can only ever * be in progress on one CPU in the system - guaranteed * by the turnstile chain lock. * * o No other operations other than MUTEX_SET_WAITERS() * and release can modify a mutex with a non-zero * owner field. * * o The result of a successful MUTEX_SET_WAITERS() call * is an unbuffered write that is immediately visible * to all other processors in the system. * * o If the holding LWP switches away, it posts a store * fence before changing curlwp, ensuring that any * overwrite of the mutex waiters flag by mutex_exit() * completes before the modification of curlwp becomes * visible to this CPU. * * o mi_switch() posts a store fence before setting curlwp * and before resuming execution of an LWP. * * o _kernel_lock() posts a store fence before setting * curcpu()->ci_biglock_wanted, and after clearing it. * This ensures that any overwrite of the mutex waiters * flag by mutex_exit() completes before the modification * of ci_biglock_wanted becomes visible. * * We now post a read memory barrier (after setting the * waiters field) and check the lock holder's status again. * Some of the possible outcomes (not an exhaustive list): * * 1. The on-CPU check returns true: the holding LWP is * running again. The lock may be released soon and * we should spin. Importantly, we can't trust the * value of the waiters flag. * * 2. The on-CPU check returns false: the holding LWP is * not running. We now have the opportunity to check * if mutex_exit() has blatted the modifications made * by MUTEX_SET_WAITERS(). * * 3. The on-CPU check returns false: the holding LWP may * or may not be running. It has context switched at * some point during our check. Again, we have the * chance to see if the waiters bit is still set or * has been overwritten. * * 4. The on-CPU check returns false: the holding LWP is * running on a CPU, but wants the big lock. It's OK * to check the waiters field in this case. * * 5. The has-waiters check fails: the mutex has been * released, the waiters flag cleared and another LWP * now owns the mutex. * * 6. The has-waiters check fails: the mutex has been * released. * * If the waiters bit is not set it's unsafe to go asleep, * as we might never be awoken. */ if ((membar_consumer(), mutex_oncpu(owner)) || (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { turnstile_exit(mtx); owner = mtx->mtx_owner; continue; } #endif /* MULTIPROCESSOR */ LOCKSTAT_START_TIMER(lsflag, slptime); turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj); LOCKSTAT_STOP_TIMER(lsflag, slptime); LOCKSTAT_COUNT(slpcnt, 1); owner = mtx->mtx_owner; } KPREEMPT_ENABLE(curlwp); LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, slpcnt, slptime); LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN, spincnt, spintime); LOCKSTAT_EXIT(lsflag); MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); MUTEX_LOCKED(mtx); }
/* * mutex_vector_enter() is called from the assembly mutex_enter() routine * if the lock is held or is not of type MUTEX_ADAPTIVE. */ void mutex_vector_enter(mutex_impl_t *lp) { kthread_id_t owner; hrtime_t sleep_time = 0; /* how long we slept */ uint_t spin_count = 0; /* how many times we spun */ cpu_t *cpup, *last_cpu; extern cpu_t *cpu_list; turnstile_t *ts; volatile mutex_impl_t *vlp = (volatile mutex_impl_t *)lp; int backoff; /* current backoff */ int backctr; /* ctr for backoff */ int sleep_count = 0; ASSERT_STACK_ALIGNED(); if (MUTEX_TYPE_SPIN(lp)) { lock_set_spl(&lp->m_spin.m_spinlock, lp->m_spin.m_minspl, &lp->m_spin.m_oldspl); return; } if (!MUTEX_TYPE_ADAPTIVE(lp)) { mutex_panic("mutex_enter: bad mutex", lp); return; } /* * Adaptive mutexes must not be acquired from above LOCK_LEVEL. * We can migrate after loading CPU but before checking CPU_ON_INTR, * so we must verify by disabling preemption and loading CPU again. */ cpup = CPU; if (CPU_ON_INTR(cpup) && !panicstr) { kpreempt_disable(); if (CPU_ON_INTR(CPU)) mutex_panic("mutex_enter: adaptive at high PIL", lp); kpreempt_enable(); } CPU_STATS_ADDQ(cpup, sys, mutex_adenters, 1); if (&plat_lock_delay) { backoff = 0; } else { backoff = BACKOFF_BASE; } for (;;) { spin: spin_count++; /* * Add an exponential backoff delay before trying again * to touch the mutex data structure. * the spin_count test and call to nulldev are to prevent * the compiler optimizer from eliminating the delay loop. */ if (&plat_lock_delay) { plat_lock_delay(&backoff); } else { for (backctr = backoff; backctr; backctr--) { if (!spin_count) (void) nulldev(); }; /* delay */ backoff = backoff << 1; /* double it */ if (backoff > BACKOFF_CAP) { backoff = BACKOFF_CAP; } SMT_PAUSE(); } if (panicstr) return; if ((owner = MUTEX_OWNER(vlp)) == NULL) { if (mutex_adaptive_tryenter(lp)) break; continue; } if (owner == curthread) mutex_panic("recursive mutex_enter", lp); /* * If lock is held but owner is not yet set, spin. * (Only relevant for platforms that don't have cas.) */ if (owner == MUTEX_NO_OWNER) continue; /* * When searching the other CPUs, start with the one where * we last saw the owner thread. If owner is running, spin. * * We must disable preemption at this point to guarantee * that the list doesn't change while we traverse it * without the cpu_lock mutex. While preemption is * disabled, we must revalidate our cached cpu pointer. */ kpreempt_disable(); if (cpup->cpu_next == NULL) cpup = cpu_list; last_cpu = cpup; /* mark end of search */ do { if (cpup->cpu_thread == owner) { kpreempt_enable(); goto spin; } } while ((cpup = cpup->cpu_next) != last_cpu); kpreempt_enable(); /* * The owner appears not to be running, so block. * See the Big Theory Statement for memory ordering issues. */ ts = turnstile_lookup(lp); MUTEX_SET_WAITERS(lp); membar_enter(); /* * Recheck whether owner is running after waiters bit hits * global visibility (above). If owner is running, spin. * * Since we are at ipl DISP_LEVEL, kernel preemption is * disabled, however we still need to revalidate our cached * cpu pointer to make sure the cpu hasn't been deleted. */ if (cpup->cpu_next == NULL) last_cpu = cpup = cpu_list; do { if (cpup->cpu_thread == owner) { turnstile_exit(lp); goto spin; } } while ((cpup = cpup->cpu_next) != last_cpu); membar_consumer(); /* * If owner and waiters bit are unchanged, block. */ if (MUTEX_OWNER(vlp) == owner && MUTEX_HAS_WAITERS(vlp)) { sleep_time -= gethrtime(); (void) turnstile_block(ts, TS_WRITER_Q, lp, &mutex_sobj_ops, NULL, NULL); sleep_time += gethrtime(); sleep_count++; } else { turnstile_exit(lp); } } ASSERT(MUTEX_OWNER(lp) == curthread); if (sleep_time != 0) { /* * Note, sleep time is the sum of all the sleeping we * did. */ LOCKSTAT_RECORD(LS_MUTEX_ENTER_BLOCK, lp, sleep_time); } /* * We do not count a sleep as a spin. */ if (spin_count > sleep_count) LOCKSTAT_RECORD(LS_MUTEX_ENTER_SPIN, lp, spin_count - sleep_count); LOCKSTAT_RECORD0(LS_MUTEX_ENTER_ACQUIRE, lp); }