void mutex_destroy(kmutex_t *mp) { mutex_impl_t *lp = (mutex_impl_t *)mp; if (lp->m_owner == 0 && !MUTEX_HAS_WAITERS(lp)) { MUTEX_DESTROY(lp); } else if (MUTEX_TYPE_SPIN(lp)) { LOCKSTAT_RECORD0(LS_MUTEX_DESTROY_RELEASE, lp); MUTEX_DESTROY(lp); } else if (MUTEX_TYPE_ADAPTIVE(lp)) { LOCKSTAT_RECORD0(LS_MUTEX_DESTROY_RELEASE, lp); if (MUTEX_OWNER(lp) != curthread) mutex_panic("mutex_destroy: not owner", lp); if (MUTEX_HAS_WAITERS(lp)) { turnstile_t *ts = turnstile_lookup(lp); turnstile_exit(lp); if (ts != NULL) mutex_panic("mutex_destroy: has waiters", lp); } MUTEX_DESTROY(lp); } else { mutex_panic("mutex_destroy: bad mutex", lp); } }
/** \brief Release the acquired mutex * * \b os_mutexUnlock calls \b pthread_mutex_unlock to release * the posix \b mutex. */ void os_mutexUnlock (os_mutex *mutex) { assert (mutex != NULL); #ifdef OSPL_STRICT_MEM assert (mutex->signature == OS_MUTEX_MAGIC_SIG); #endif if (mutex->name[0] != '\0') { STATUS status; SEM_ID id; #if 1 /* NB: id lookup in cache is for debugging reasons */ id = get_mutex (mutex); #else id = pa_ldvoidp (&mutex->info.id); #endif cas_mutex_info_id (mutex, id, 0); status = semGive (id); if (status != OK) { mutex_panic("unlock: semGive failed", status); } } else { int res = pthread_mutex_unlock (&mutex->info.posix_mutex); if (res != 0) { mutex_panic ("unlock: semGive failed", res); } } }
static SEM_ID get_mutex (os_mutex *mutex) { SEM_ID result; int options; semTake (cache_lock, WAIT_FOREVER); result = (SEM_ID) ut_get (cache, mutex); if (!result) { /* In single process mode only "private" variables are required */ options = (SEM_Q_PRIORITY | SEM_DELETE_SAFE); /* Set option for priority inheritance if feature is enabled */ if (ospl_mtx_prio_inherit) { options = options | SEM_INVERSION_SAFE; } result = semOpen(mutex->name, SEM_TYPE_MUTEX, SEM_EMPTY, options, 0, NULL); if (result) { (void) ut_tableInsert ((ut_table) cache, mutex, result); } else { mutex_panic ("get_mutex", 0); } } semGive (cache_lock); return result; }
/** \brief Destroy the mutex * * \b os_mutexDestroy calls \b pthread_mutex_destroy to destroy the * posix \b mutex. */ void os_mutexDestroy (os_mutex *mutex) { assert (mutex != NULL); #ifdef OSPL_STRICT_MEM assert (mutex->signature == OS_MUTEX_MAGIC_SIG); #endif if (mutex->name[0] != '\0') { SEM_ID id; /* shared mutex */ id = ut_get (cache, mutex); cas_mutex_info_id (mutex, 0, invalid_sem_id); if (id) { semClose (id); semTake(cache_lock, WAIT_FOREVER); ut_remove (cache, mutex); semGive(cache_lock); } } else { int res = pthread_mutex_destroy (&mutex->info.posix_mutex); if (res != 0) { mutex_panic ("pthread_mutex_destroy failed", res); } } #ifdef OSPL_STRICT_MEM mutex->signature = 0; #endif }
os_result os_mutexInit (os_mutex *mutex, const os_mutexAttr *mutexAttr) { os_mutexAttr defAttr; os_result rv; int options; SEM_ID id; assert (mutex != NULL); #ifdef OSPL_STRICT_MEM assert (mutex->signature != OS_MUTEX_MAGIC_SIG); #endif if(!mutexAttr) { os_mutexAttrInit(&defAttr); mutexAttr = &defAttr; } options = (SEM_Q_PRIORITY | SEM_DELETE_SAFE); /* Set option for priority inheritance if feature is enabled */ if (ospl_mtx_prio_inherit) { options = options | SEM_INVERSION_SAFE; } if (mutexAttr->scopeAttr == OS_SCOPE_SHARED) { /* create named mutex using shared address as unique name */ snprintf (mutex->name, sizeof (mutex->name), "/%lu", (os_uint32) mutex); id = semOpen (mutex->name, SEM_TYPE_MUTEX, SEM_EMPTY, options, OM_CREATE|OM_EXCL, NULL); /* As it may exist from a previous run. remove leftover semaphore object */ if (id != NULL) { /* a fresh one, initialise it and cache the SEM_ID */ pa_st32 ((void *) &mutex->info.id, 0); semTake (cache_lock, WAIT_FOREVER); (void) ut_tableInsert ((ut_table) cache, mutex, id); semGive (cache_lock); } else { /* re-using an old one: it should have been destroyed and hence should have a marker in info.id */ id = get_mutex (mutex); if (id != NULL) { semTake (id, WAIT_FOREVER); semGive (id); /*cas_mutex_info_id ((void*)&mutex->info.id, invalid_sem_id, 0);*/ } else { mutex_panic ("init: attempt to reuse semaphore currently in use", 0); } } rv = (id != NULL) ? os_resultSuccess : os_resultFail; } else { int result; mutex->name[0] = '\0'; /* TODO attrs */ result = pthread_mutex_init(&mutex->info.posix_mutex, NULL); rv = (result == 0) ? os_resultSuccess : os_resultFail; } #ifdef OSPL_STRICT_MEM mutex->signature = OS_MUTEX_MAGIC_SIG; #endif return rv; }
static void cas_mutex_info_id (os_mutex *mutex, SEM_ID expected, SEM_ID id) { SEM_ID cached_id; do { cached_id = pa_ldvoidp ((void*) &mutex->info.id); if (cached_id != expected) { mutex_panic ("sem id mismatch in cas_mutex_info_id", 0); } } while (!pa_casvoidp ((void*) &mutex->info.id, expected, id)); }
/** \brief Try to acquire the mutex, immediately return if the mutex * is already acquired by another thread * * \b os_mutexTryLock calls \b pthread_mutex_trylock to acquire * the posix \b mutex. */ os_result os_mutexTryLock (os_mutex *mutex) { os_result rv = os_resultFail; assert (mutex != NULL); #ifdef OSPL_STRICT_MEM assert (mutex->signature == OS_MUTEX_MAGIC_SIG); #endif if (mutex->name[0] != '\0') { STATUS result; SEM_ID id; id = get_mutex (mutex); result = semTake(id, NO_WAIT); if (result == OK) { cas_mutex_info_id (mutex, 0, id); rv = os_resultSuccess; } else { if (os_getErrno() == S_objLib_OBJ_UNAVAILABLE) { rv = os_resultBusy; } else { mutex_panic ("trylock: semTake failed", result); } } } else { int res = pthread_mutex_trylock (&mutex->info.posix_mutex); if (res == 0) { rv = os_resultSuccess; } else if (res == EBUSY) { rv = os_resultBusy; } else { mutex_panic ("pthread_mutex_trylock failed", res); } } return rv; }
/* * mutex_vector_tryenter() is called from the assembly mutex_tryenter() * routine if the lock is held or is not of type MUTEX_ADAPTIVE. */ int mutex_vector_tryenter(mutex_impl_t *lp) { int s; if (MUTEX_TYPE_ADAPTIVE(lp)) return (0); /* we already tried in assembly */ if (!MUTEX_TYPE_SPIN(lp)) { mutex_panic("mutex_tryenter: bad mutex", lp); return (0); } s = splr(lp->m_spin.m_minspl); if (lock_try(&lp->m_spin.m_spinlock)) { lp->m_spin.m_oldspl = (ushort_t)s; return (1); } splx(s); return (0); }
/* * mutex_vector_exit() is called from mutex_exit() if the lock is not * adaptive, has waiters, or is not owned by the current thread (panic). */ void mutex_vector_exit(mutex_impl_t *lp) { turnstile_t *ts; if (MUTEX_TYPE_SPIN(lp)) { lock_clear_splx(&lp->m_spin.m_spinlock, lp->m_spin.m_oldspl); return; } if (MUTEX_OWNER(lp) != curthread) { mutex_panic("mutex_exit: not owner", lp); return; } ts = turnstile_lookup(lp); MUTEX_CLEAR_LOCK_AND_WAITERS(lp); if (ts == NULL) turnstile_exit(lp); else turnstile_wakeup(ts, TS_WRITER_Q, ts->ts_waiters, NULL); LOCKSTAT_RECORD0(LS_MUTEX_EXIT_RELEASE, lp); }
void os_mutexLock (os_mutex *mutex) { if (os_mutexLock_s (mutex) != os_resultSuccess) { mutex_panic ("mutexLock failed", 0); } }
/* * mutex_vector_enter() is called from the assembly mutex_enter() routine * if the lock is held or is not of type MUTEX_ADAPTIVE. */ void mutex_vector_enter(mutex_impl_t *lp) { kthread_id_t owner; hrtime_t sleep_time = 0; /* how long we slept */ uint_t spin_count = 0; /* how many times we spun */ cpu_t *cpup, *last_cpu; extern cpu_t *cpu_list; turnstile_t *ts; volatile mutex_impl_t *vlp = (volatile mutex_impl_t *)lp; int backoff; /* current backoff */ int backctr; /* ctr for backoff */ int sleep_count = 0; ASSERT_STACK_ALIGNED(); if (MUTEX_TYPE_SPIN(lp)) { lock_set_spl(&lp->m_spin.m_spinlock, lp->m_spin.m_minspl, &lp->m_spin.m_oldspl); return; } if (!MUTEX_TYPE_ADAPTIVE(lp)) { mutex_panic("mutex_enter: bad mutex", lp); return; } /* * Adaptive mutexes must not be acquired from above LOCK_LEVEL. * We can migrate after loading CPU but before checking CPU_ON_INTR, * so we must verify by disabling preemption and loading CPU again. */ cpup = CPU; if (CPU_ON_INTR(cpup) && !panicstr) { kpreempt_disable(); if (CPU_ON_INTR(CPU)) mutex_panic("mutex_enter: adaptive at high PIL", lp); kpreempt_enable(); } CPU_STATS_ADDQ(cpup, sys, mutex_adenters, 1); if (&plat_lock_delay) { backoff = 0; } else { backoff = BACKOFF_BASE; } for (;;) { spin: spin_count++; /* * Add an exponential backoff delay before trying again * to touch the mutex data structure. * the spin_count test and call to nulldev are to prevent * the compiler optimizer from eliminating the delay loop. */ if (&plat_lock_delay) { plat_lock_delay(&backoff); } else { for (backctr = backoff; backctr; backctr--) { if (!spin_count) (void) nulldev(); }; /* delay */ backoff = backoff << 1; /* double it */ if (backoff > BACKOFF_CAP) { backoff = BACKOFF_CAP; } SMT_PAUSE(); } if (panicstr) return; if ((owner = MUTEX_OWNER(vlp)) == NULL) { if (mutex_adaptive_tryenter(lp)) break; continue; } if (owner == curthread) mutex_panic("recursive mutex_enter", lp); /* * If lock is held but owner is not yet set, spin. * (Only relevant for platforms that don't have cas.) */ if (owner == MUTEX_NO_OWNER) continue; /* * When searching the other CPUs, start with the one where * we last saw the owner thread. If owner is running, spin. * * We must disable preemption at this point to guarantee * that the list doesn't change while we traverse it * without the cpu_lock mutex. While preemption is * disabled, we must revalidate our cached cpu pointer. */ kpreempt_disable(); if (cpup->cpu_next == NULL) cpup = cpu_list; last_cpu = cpup; /* mark end of search */ do { if (cpup->cpu_thread == owner) { kpreempt_enable(); goto spin; } } while ((cpup = cpup->cpu_next) != last_cpu); kpreempt_enable(); /* * The owner appears not to be running, so block. * See the Big Theory Statement for memory ordering issues. */ ts = turnstile_lookup(lp); MUTEX_SET_WAITERS(lp); membar_enter(); /* * Recheck whether owner is running after waiters bit hits * global visibility (above). If owner is running, spin. * * Since we are at ipl DISP_LEVEL, kernel preemption is * disabled, however we still need to revalidate our cached * cpu pointer to make sure the cpu hasn't been deleted. */ if (cpup->cpu_next == NULL) last_cpu = cpup = cpu_list; do { if (cpup->cpu_thread == owner) { turnstile_exit(lp); goto spin; } } while ((cpup = cpup->cpu_next) != last_cpu); membar_consumer(); /* * If owner and waiters bit are unchanged, block. */ if (MUTEX_OWNER(vlp) == owner && MUTEX_HAS_WAITERS(vlp)) { sleep_time -= gethrtime(); (void) turnstile_block(ts, TS_WRITER_Q, lp, &mutex_sobj_ops, NULL, NULL); sleep_time += gethrtime(); sleep_count++; } else { turnstile_exit(lp); } } ASSERT(MUTEX_OWNER(lp) == curthread); if (sleep_time != 0) { /* * Note, sleep time is the sum of all the sleeping we * did. */ LOCKSTAT_RECORD(LS_MUTEX_ENTER_BLOCK, lp, sleep_time); } /* * We do not count a sleep as a spin. */ if (spin_count > sleep_count) LOCKSTAT_RECORD(LS_MUTEX_ENTER_SPIN, lp, spin_count - sleep_count); LOCKSTAT_RECORD0(LS_MUTEX_ENTER_ACQUIRE, lp); }