C99INLINE
#endif
static void
plasma_spin_pause_yield_adaptive (const uint32_t distance, const int callcount)
{
/* simplistic backoff employing call count and distance to acquire fair lock
* (might be adjusted to pause multiple times for slightly larger input
* distances, especially on machines with larger numbers of CPUs/cores/SMT,
* but be sure to always yield quickly/immediately if distance >= nprocs)
*
* callers must initialize nshift and nprocs prior to calling this routine
* (initialization avoided here since code might be inlined in tight loop)
* (ok if a few threads run the initialization; results will be same)
* if (__builtin_expect( (!nprocs), 0))
* plasma_spin_nprocs_init();
*
* (nshift == 0 for one core; always yield CPU
* nshift == 1 for two cores; test distance <= nshift)
*/
if (callcount < 128 && distance <= nshift) {
plasma_spin_pause(); /* brief pause if almost our turn */
}
else {
plasma_spin_yield(); /* yield CPU if highly contended */
}
}
bool
plasma_spin_lock_acquire_spinloop (plasma_spin_lock_t * const spin)
{
uint32_t * const lck = &spin->lck;
do {
while (plasma_atomic_load_explicit(lck, memory_order_relaxed)) {
plasma_spin_pause();
}
} while (!plasma_atomic_lock_acquire(lck)); /*(includes barrier)*/
return true;
}
bool
plasma_spin_lock_acquire_spinloop (plasma_spin_lock_t * const spin)
{
uint32_t * const lck = &spin->lck;
do {
while (plasma_atomic_ld_nopt_T(uint32_t *, lck)) {
plasma_spin_pause();
}
} while (!plasma_atomic_lock_acquire(lck)); /*(includes barrier)*/
return true;
}
bool
plasma_spin_lock_acquire_spindecay (plasma_spin_lock_t * const spin,
int pause1, int pause32, int yield)
{
/* ((uint32_t *) cast also works for Apple OSSpinLock, which is int32_t) */
uint32_t * const lck = (uint32_t *)&spin->lck;
do {
while (plasma_atomic_load_explicit(lck, memory_order_relaxed)) {
if (pause1) {
--pause1;
plasma_spin_pause();
}
else if (pause32) {
int i = 4;
--pause32;
do {
plasma_spin_pause();
plasma_spin_pause();
plasma_spin_pause();
plasma_spin_pause();
plasma_spin_pause();
plasma_spin_pause();
plasma_spin_pause();
plasma_spin_pause();
} while (--i);
}
else if (yield) {
--yield;
plasma_spin_yield();
}
else
return false;
}
} while (!plasma_atomic_lock_acquire(lck)); /*(includes barrier)*/
return true;
}