void* cas_thread( object_t thread, void* arg )
{
int loop = 0;
cas_value_t val = *(cas_value_t*)arg;
thread_sleep( 10 );
while( !thread_should_terminate( thread ) && ( loop < 65535 ) )
{
while( !atomic_cas32( &val_32, val.val_32, 0 ) )
thread_yield();
while( !atomic_cas32( &val_32, 0, val.val_32 ) )
thread_yield();
while( !atomic_cas64( &val_64, val.val_64, 0 ) )
thread_yield();
while( !atomic_cas64( &val_64, 0, val.val_64 ) )
thread_yield();
while( !atomic_cas_ptr( &val_ptr, val.val_ptr, 0 ) )
thread_yield();
while( !atomic_cas_ptr( &val_ptr, 0, val.val_ptr ) )
thread_yield();
++loop;
thread_yield();
}
return 0;
}
static void*
cas_thread(void* arg) {
unsigned int loop = 0;
cas_value_t val = *(cas_value_t*)arg;
thread_sleep(10);
while (!thread_try_wait(0) && (loop < 65535)) {
while (!atomic_cas32(&val_32, val.val_32, 0))
thread_yield();
while (!atomic_cas32(&val_32, 0, val.val_32))
thread_yield();
while (!atomic_cas64(&val_64, val.val_64, 0))
thread_yield();
while (!atomic_cas64(&val_64, 0, val.val_64))
thread_yield();
while (!atomic_cas_ptr(&val_ptr, val.val_ptr, 0))
thread_yield();
while (!atomic_cas_ptr(&val_ptr, 0, val.val_ptr))
thread_yield();
++loop;
thread_yield();
}
return 0;
}
// Perform atomic 'compare and swap' operation on the pointer.
// The pointer is compared to 'cmp' argument and if they are
// equal, its value is set to 'val'. Old value of the pointer
// is returned.
inline T *cas (T *cmp_, T *val_)
{
#if defined XS_ATOMIC_PTR_WINDOWS
return (T*) InterlockedCompareExchangePointer (
(volatile PVOID*) &ptr, val_, cmp_);
#elif defined XS_ATOMIC_PTR_ATOMIC_H
return (T*) atomic_cas_ptr (&ptr, cmp_, val_);
#elif defined XS_ATOMIC_PTR_X86
T *old;
__asm__ volatile (
"lock; cmpxchg %2, %3"
: "=a" (old), "=m" (ptr)
: "r" (val_), "m" (ptr), "0" (cmp_)
: "cc");
return old;
#elif defined XS_ATOMIC_PTR_MUTEX
sync.lock ();
T *old = (T*) ptr;
if (ptr == cmp_)
ptr = val_;
sync.unlock ();
return old;
#else
#error atomic_ptr is not implemented for this platform
#endif
}
int
mtx_enter_try(struct mutex *mtx)
{
struct cpu_info *owner, *ci = curcpu();
int s;
if (mtx->mtx_wantipl != IPL_NONE)
s = splraise(mtx->mtx_wantipl);
owner = atomic_cas_ptr(&mtx->mtx_owner, NULL, ci);
#ifdef DIAGNOSTIC
if (__predict_false(owner == ci))
panic("mtx %p: locking against myself", mtx);
#endif
if (owner == NULL) {
if (mtx->mtx_wantipl != IPL_NONE)
mtx->mtx_oldipl = s;
#ifdef DIAGNOSTIC
ci->ci_mutex_level++;
#endif
membar_enter();
return (1);
}
if (mtx->mtx_wantipl != IPL_NONE)
splx(s);
return (0);
}
/* Using the CAS style unlocks, since the usurper recovery is a real pain in the
* ass */
void __mcs_pdro_unlock(struct mcs_pdro_lock *lock, struct mcs_pdro_qnode *qnode)
{
uint32_t a_tail_vcoreid;
/* Check if someone is already waiting on us to unlock */
if (qnode->next == 0) {
cmb(); /* no need for CPU mbs, since there's an atomic_cas() */
/* If we're still the lock, just swap it with 0 (unlock) and return */
if (atomic_cas_ptr((void**)&lock->lock, qnode, 0))
return;
/* Read in the tail (or someone who recently was the tail, but could now
* be farther up the chain), in prep for our spinning. */
a_tail_vcoreid = ACCESS_ONCE(lock->lock->vcoreid);
/* We failed, someone is there and we are some (maybe a different)
* thread's pred. Since someone else was waiting, they should have made
* themselves our next. Spin (very briefly!) til it happens. */
while (qnode->next == 0) {
/* We need to get our next to run, but we don't know who they are.
* If we make sure a tail is running, that will percolate up to make
* sure our qnode->next is running */
ensure_vcore_runs(a_tail_vcoreid);
cpu_relax();
}
/* Alpha wants a read_barrier_depends() here */
/* Now that we have a next, unlock them */
qnode->next->locked = 0;
} else {
/* mb()s necessary since we didn't call an atomic_swap() */
wmb(); /* need to make sure any previous writes don't pass unlocking */
rwmb(); /* need to make sure any reads happen before the unlocking */
/* simply unlock whoever is next */
qnode->next->locked = 0;
}
}
static inline uintptr_t
rw_cas(pthread_rwlock_t *ptr, uintptr_t o, uintptr_t n)
{
return (uintptr_t)atomic_cas_ptr(&ptr->ptr_owner, (void *)o,
(void *)n);
}
inline bool compare_and_swap(volatile PtrT *ptr, const ValueT& comp_val, const ValueT& exchange_val)
{
#if ELEVELDB_IS_SOLARIS
return (comp_val==atomic_cas_ptr(ptr, comp_val, exchange_val));
#else
return __sync_bool_compare_and_swap(ptr, comp_val, exchange_val);
#endif
}
static inline uintptr_t
rw_cas(krwlock_t *rw, uintptr_t o, uintptr_t n)
{
RW_INHERITDEBUG(n, o);
return (uintptr_t)atomic_cas_ptr((volatile void *)&rw->rw_owner,
(void *)o, (void *)n);
}
// Perform atomic 'compare and swap' operation on the pointer.
// The pointer is compared to 'cmp' argument and if they are
// equal, its value is set to 'val'. Old value of the pointer
// is returned.
inline T *cas (T *cmp_, T *val_)
{
#if defined ZMQ_ATOMIC_PTR_WINDOWS
return (T*) InterlockedCompareExchangePointer (
(volatile PVOID*) &ptr, val_, cmp_);
#elif defined ZMQ_ATOMIC_PTR_INTRINSIC
T *old = cmp_;
__atomic_compare_exchange_n (&ptr, (volatile T**) &old, val_, false,
__ATOMIC_RELEASE, __ATOMIC_ACQUIRE);
return old;
#elif defined ZMQ_ATOMIC_PTR_CXX11
ptr.compare_exchange_strong(cmp_, val_, std::memory_order_acq_rel);
return cmp_;
#elif defined ZMQ_ATOMIC_PTR_ATOMIC_H
return (T*) atomic_cas_ptr (&ptr, cmp_, val_);
#elif defined ZMQ_ATOMIC_PTR_TILE
return (T*) arch_atomic_val_compare_and_exchange (&ptr, cmp_, val_);
#elif defined ZMQ_ATOMIC_PTR_X86
T *old;
__asm__ volatile (
"lock; cmpxchg %2, %3"
: "=a" (old), "=m" (ptr)
: "r" (val_), "m" (ptr), "0" (cmp_)
: "cc");
return old;
#elif defined ZMQ_ATOMIC_PTR_ARM
T *old;
unsigned int flag;
__asm__ volatile (
" dmb sy\n\t"
"1: ldrex %1, [%3]\n\t"
" mov %0, #0\n\t"
" teq %1, %4\n\t"
" it eq\n\t"
" strexeq %0, %5, [%3]\n\t"
" teq %0, #0\n\t"
" bne 1b\n\t"
" dmb sy\n\t"
: "=&r"(flag), "=&r"(old), "+Qo"(ptr)
: "r"(&ptr), "r"(cmp_), "r"(val_)
: "cc");
return old;
#elif defined ZMQ_ATOMIC_PTR_MUTEX
sync.lock ();
T *old = (T*) ptr;
if (ptr == cmp_)
ptr = val_;
sync.unlock ();
return old;
#else
#error atomic_ptr is not implemented for this platform
#endif
}
static void
rw_panic(char *msg, rwlock_impl_t *lp)
{
if (panicstr)
return;
if (atomic_cas_ptr(&panic_rwlock_addr, NULL, lp) == NULL)
panic_rwlock = *lp;
panic("%s, lp=%p wwwh=%lx thread=%p",
msg, (void *)lp, panic_rwlock.rw_wwwh, (void *)curthread);
}
void BLI_linklist_lockfree_insert(LockfreeLinkList *list, LockfreeLinkNode *node)
{
/* Based on:
*
* John D. Valois
* Implementing Lock-Free Queues
*
* http://people.csail.mit.edu/bushl2/rpi/portfolio/lockfree-grape/documents/lock-free-linked-lists.pdf
*/
bool keep_working;
LockfreeLinkNode *tail_node;
node->next = NULL;
do {
tail_node = list->tail;
keep_working = (atomic_cas_ptr((void **)&tail_node->next, NULL, node) != NULL);
if (keep_working) {
atomic_cas_ptr((void **)&list->tail, tail_node, tail_node->next);
}
} while (keep_working);
atomic_cas_ptr((void **)&list->tail, tail_node, node);
}
/* ARGSUSED */
static int
ksyms_open(dev_t *devp, int flag, int otyp, struct cred *cred)
{
minor_t clone;
size_t size = 0;
size_t realsize;
char *addr;
void *hptr = NULL;
ksyms_buflist_hdr_t hdr;
bzero(&hdr, sizeof (struct ksyms_buflist_hdr));
list_create(&hdr.blist, PAGESIZE,
offsetof(ksyms_buflist_t, buflist_node));
if (getminor(*devp) != 0)
return (ENXIO);
for (;;) {
realsize = ksyms_snapshot(ksyms_bcopy, hptr, size);
if (realsize <= size)
break;
size = realsize;
size = ksyms_buflist_alloc(&hdr, size);
if (size == 0)
return (ENOMEM);
hptr = (void *)&hdr;
}
addr = ksyms_mapin(&hdr, realsize);
ksyms_buflist_free(&hdr);
if (addr == NULL)
return (EOVERFLOW);
/*
* Reserve a clone entry. Note that we don't use clone 0
* since that's the "real" minor number.
*/
for (clone = 1; clone < nksyms_clones; clone++) {
if (atomic_cas_ptr(&ksyms_clones[clone].ksyms_base, 0, addr) ==
0) {
ksyms_clones[clone].ksyms_size = realsize;
*devp = makedevice(getemajor(*devp), clone);
(void) ddi_prop_update_int(*devp, ksyms_devi,
"size", realsize);
modunload_disable();
return (0);
}
}
cmn_err(CE_NOTE, "ksyms: too many open references");
(void) as_unmap(curproc->p_as, addr, roundup(realsize, PAGESIZE));
return (ENXIO);
}
/*
* cms_init entry point.
*
* This module provides broad model-specific support for AMD families
* 0x6, 0xf and 0x10. Future families will have to be evaluated once their
* documentation is available.
*/
int
authamd_init(cmi_hdl_t hdl, void **datap)
{
uint_t chipid = cmi_hdl_chipid(hdl);
uint_t procnodeid = cmi_hdl_procnodeid(hdl);
struct authamd_nodeshared *sp, *osp;
uint_t family = cmi_hdl_family(hdl);
uint32_t rev = cmi_hdl_chiprev(hdl);
authamd_data_t *authamd;
uint64_t cap;
if (authamd_ms_support_disable ||
!authamd_supported(hdl))
return (ENOTSUP);
if (!is_x86_feature(x86_featureset, X86FSET_MCA))
return (ENOTSUP);
if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) != CMI_SUCCESS)
return (ENOTSUP);
if (!(cap & MCG_CAP_CTL_P))
return (ENOTSUP);
authamd = *datap = kmem_zalloc(sizeof (authamd_data_t), KM_SLEEP);
cmi_hdl_hold(hdl); /* release in fini */
authamd->amd_hdl = hdl;
if ((sp = authamd_shared[procnodeid]) == NULL) {
sp = kmem_zalloc(sizeof (struct authamd_nodeshared), KM_SLEEP);
sp->ans_chipid = chipid;
sp->ans_procnodeid = procnodeid;
sp->ans_family = family;
sp->ans_rev = rev;
membar_producer();
osp = atomic_cas_ptr(&authamd_shared[procnodeid], NULL, sp);
if (osp != NULL) {
kmem_free(sp, sizeof (struct authamd_nodeshared));
sp = osp;
}
}
authamd->amd_shared = sp;
return (0);
}
void __mcs_pdr_unlock(struct mcs_pdr_lock *lock, struct mcs_pdr_qnode *qnode)
{
uint32_t a_tail_vcoreid;
struct mcs_pdr_qnode *qnode0 = qnode - vcore_id();
/* Check if someone is already waiting on us to unlock */
if (qnode->next == 0) {
cmb(); /* no need for CPU mbs, since there's an atomic_cas() */
/* If we're still the lock, just swap it with 0 (unlock) and return */
if (atomic_cas_ptr((void**)&lock->lock, qnode, 0)) {
/* This is racy with the new lockholder. it's possible that we'll
* clobber their legit write, though it doesn't actually hurt
* correctness. it'll get sorted out on the next unlock. */
lock->lockholder_vcoreid = MCSPDR_NO_LOCKHOLDER;
return;
}
/* Get the tail (or someone who recently was the tail, but could now
* be farther up the chain), in prep for our spinning. Could do an
* ACCESS_ONCE on lock->lock */
a_tail_vcoreid = lock->lock - qnode0;
/* We failed, someone is there and we are some (maybe a different)
* thread's pred. Since someone else was waiting, they should have made
* themselves our next. Spin (very briefly!) til it happens. */
while (qnode->next == 0) {
/* We need to get our next to run, but we don't know who they are.
* If we make sure a tail is running, that will percolate up to make
* sure our qnode->next is running */
ensure_vcore_runs(a_tail_vcoreid);
cpu_relax();
}
/* Alpha wants a read_barrier_depends() here */
lock->lockholder_vcoreid = qnode->next - qnode0;
wmb(); /* order the vcoreid write before the unlock */
qnode->next->locked = 0;
} else {
/* Note we're saying someone else is the lockholder, though we still are
* the lockholder until we unlock the next qnode. Our next knows that
* if it sees itself is the lockholder, that it needs to make sure we
* run. */
lock->lockholder_vcoreid = qnode->next - qnode0;
/* mb()s necessary since we didn't call an atomic_swap() */
wmb(); /* need to make sure any previous writes don't pass unlocking */
rwmb(); /* need to make sure any reads happen before the unlocking */
/* simply unlock whoever is next */
qnode->next->locked = 0;
}
}
void*
_allocate_thread_local_block(size_t size) {
void* block = memory_allocate(0, size, 0, MEMORY_PERSISTENT | MEMORY_ZERO_INITIALIZED);
for (int i = 0; i < 1024; ++i) {
if (!atomic_loadptr(&_thread_local_blocks[i].block)) {
if (atomic_cas_ptr(&_thread_local_blocks[i].block, block, 0)) {
_thread_local_blocks[i].thread = thread_id();
return block;
}
}
}
log_warnf(0, WARNING_MEMORY,
STRING_CONST("Unable to locate thread local memory block slot, will leak %" PRIsize " bytes"),
size);
return block;
}
static evchan_t *
bind_channel(boolean_t priv, fmev_pri_t pri)
{
evchan_t **evcpp = &CHAN_BINDING(priv, pri);
evchan_t *evc;
if (*evcpp != NULL)
return (*evcpp);
if (sysevent_evc_bind(CHAN_NAME(priv, pri), &evc,
EVCH_CREAT | CHAN_FLAGS(priv, pri)) != 0)
return (NULL);
if (atomic_cas_ptr(evcpp, NULL, evc) != NULL)
(void) sysevent_evc_unbind(evc);
return (*evcpp);
}
/*
* Fetch a /dev/u?random context's CPRNG, or create and save one if
* necessary.
*/
static struct cprng_strong *
rnd_ctx_cprng(struct rnd_ctx *ctx)
{
struct cprng_strong *cprng, *tmp = NULL;
/* Fast path: if someone has already allocated a CPRNG, use it. */
cprng = ctx->rc_cprng;
if (__predict_true(cprng != NULL)) {
/* Make sure the CPU hasn't prefetched cprng's guts. */
membar_consumer();
goto out;
}
/* Slow path: create a CPRNG. Allocate before taking locks. */
char name[64];
struct lwp *const l = curlwp;
(void)snprintf(name, sizeof(name), "%d %"PRIu64" %u",
(int)l->l_proc->p_pid, l->l_ncsw, l->l_cpticks);
const int flags = (ctx->rc_hard? (CPRNG_USE_CV | CPRNG_HARD) :
(CPRNG_INIT_ANY | CPRNG_REKEY_ANY));
tmp = cprng_strong_create(name, IPL_NONE, flags);
/* Publish cprng's guts before the pointer to them. */
membar_producer();
/* Attempt to publish tmp, unless someone beat us. */
cprng = atomic_cas_ptr(&ctx->rc_cprng, NULL, tmp);
if (__predict_false(cprng != NULL)) {
/* Make sure the CPU hasn't prefetched cprng's guts. */
membar_consumer();
goto out;
}
/* Published. Commit tmp. */
cprng = tmp;
tmp = NULL;
out: if (tmp != NULL)
cprng_strong_destroy(tmp);
KASSERT(cprng != NULL);
return cprng;
}
static void _memory_tracker_track( void* addr, uint64_t size )
{
if( addr ) do
{
int32_t tag = atomic_exchange_and_add32( &_memory_tag_next, 1 );
if( tag >= MAX_CONCURRENT_ALLOCATIONS )
{
int32_t newtag = tag % MAX_CONCURRENT_ALLOCATIONS;
atomic_cas32( &_memory_tag_next, newtag, tag + 1 );
tag = newtag;
}
if( !_memory_tags[ tag ].address && atomic_cas_ptr( &_memory_tags[ tag ].address, addr, 0 ) )
{
_memory_tags[ tag ].size = (uintptr_t)size;
stacktrace_capture( _memory_tags[ tag ].trace, 14, 3 );
hashtable_set( _memory_table, (uintptr_t)addr, (uintptr_t)( tag + 1 ) );
return;
}
} while( true );
}
/*
* Our cmi_init entry point, called during startup of each cpu instance.
*/
int
gcpu_init(cmi_hdl_t hdl, void **datap)
{
uint_t chipid = cmi_hdl_chipid(hdl);
struct gcpu_chipshared *sp, *osp;
gcpu_data_t *gcpu;
if (gcpu_disable || chipid >= GCPU_MAX_CHIPID)
return (ENOTSUP);
/*
* Allocate the state structure for this cpu. We will only
* allocate the bank logout areas in gcpu_mca_init once we
* know how many banks there are.
*/
gcpu = *datap = kmem_zalloc(sizeof (gcpu_data_t), KM_SLEEP);
cmi_hdl_hold(hdl); /* release in gcpu_fini */
gcpu->gcpu_hdl = hdl;
/*
* Allocate a chipshared structure if no sibling cpu has already
* allocated it, but allow for the fact that a sibling core may
* be starting up in parallel.
*/
if ((sp = gcpu_shared[chipid]) == NULL) {
sp = kmem_zalloc(sizeof (struct gcpu_chipshared), KM_SLEEP);
osp = atomic_cas_ptr(&gcpu_shared[chipid], NULL, sp);
if (osp == NULL) {
mutex_init(&sp->gcpus_poll_lock, NULL, MUTEX_DRIVER,
NULL);
mutex_init(&sp->gcpus_cfglock, NULL, MUTEX_DRIVER,
NULL);
} else {
kmem_free(sp, sizeof (struct gcpu_chipshared));
sp = osp;
}
}
gcpu->gcpu_shared = sp;
return (0);
}
static void*
_atomic_allocate_linear(size_t chunksize) {
void* old_head;
void* new_head;
void* return_pointer = 0;
do {
old_head = atomic_loadptr(&_memory_temporary.head);
new_head = pointer_offset(old_head, chunksize);
return_pointer = old_head;
if (new_head > _memory_temporary.end) {
new_head = pointer_offset(_memory_temporary.storage, chunksize);
return_pointer = _memory_temporary.storage;
}
}
while (!atomic_cas_ptr(&_memory_temporary.head, new_head, old_head));
return return_pointer;
}
struct uidinfo *
uid_find(uid_t uid)
{
struct uidinfo *uip, *uip_first, *newuip;
struct uihashhead *uipp;
uipp = UIHASH(uid);
newuip = NULL;
/*
* To make insertion atomic, abstraction of SLIST will be violated.
*/
uip_first = uipp->slh_first;
again:
SLIST_FOREACH(uip, uipp, ui_hash) {
if (uip->ui_uid != uid)
continue;
if (newuip != NULL)
kmem_free(newuip, sizeof(*newuip));
return uip;
}
if (newuip == NULL)
newuip = kmem_zalloc(sizeof(*newuip), KM_SLEEP);
newuip->ui_uid = uid;
/*
* If atomic insert is unsuccessful, another thread might be
* allocated this 'uid', thus full re-check is needed.
*/
newuip->ui_hash.sle_next = uip_first;
membar_producer();
uip = atomic_cas_ptr(&uipp->slh_first, uip_first, newuip);
if (uip != uip_first) {
uip_first = uip;
goto again;
}
return newuip;
}
void
vpanic(const char *fmt, va_list ap)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci, *oci;
int bootopt;
static char scratchstr[256]; /* stores panic message */
spldebug_stop();
if (lwp0.l_cpu && curlwp) {
/*
* Disable preemption. If already panicing on another CPU, sit
* here and spin until the system is rebooted. Allow the CPU that
* first paniced to panic again.
*/
kpreempt_disable();
ci = curcpu();
oci = atomic_cas_ptr((void *)&paniccpu, NULL, ci);
if (oci != NULL && oci != ci) {
/* Give interrupts a chance to try and prevent deadlock. */
for (;;) {
#ifndef _RUMPKERNEL /* XXXpooka: temporary build fix, see kern/40505 */
DELAY(10);
#endif /* _RUMPKERNEL */
}
}
/*
* Convert the current thread to a bound thread and prevent all
* CPUs from scheduling unbound jobs. Do so without taking any
* locks.
*/
curlwp->l_pflag |= LP_BOUND;
for (CPU_INFO_FOREACH(cii, ci)) {
ci->ci_schedstate.spc_flags |= SPCF_OFFLINE;
}
}
/* CAS style mcs locks, kept around til we use them. We're using the
* usurper-style, since RISCV doesn't have a real CAS (yet?). */
void mcs_lock_unlock_cas(struct mcs_lock *lock, struct mcs_lock_qnode *qnode)
{
/* Check if someone is already waiting on us to unlock */
if (qnode->next == 0) {
cmb(); /* no need for CPU mbs, since there's an atomic_cas() */
/* If we're still the lock, just swap it with 0 (unlock) and return */
if (atomic_cas_ptr((void**)&lock->lock, qnode, 0))
return;
/* We failed, someone is there and we are some (maybe a different)
* thread's pred. Since someone else was waiting, they should have made
* themselves our next. Spin (very briefly!) til it happens. */
while (qnode->next == 0)
cpu_relax();
/* Alpha wants a read_barrier_depends() here */
/* Now that we have a next, unlock them */
qnode->next->locked = 0;
} else {
/* mb()s necessary since we didn't call an atomic_swap() */
wmb(); /* need to make sure any previous writes don't pass unlocking */
rwmb(); /* need to make sure any reads happen before the unlocking */
/* simply unlock whoever is next */
qnode->next->locked = 0;
}
}
/*
* Schedule a CPU. This optimizes for the case where we schedule
* the same thread often, and we have nCPU >= nFrequently-Running-Thread
* (where CPU is virtual rump cpu, not host CPU).
*/
void
rump_schedule_cpu_interlock(struct lwp *l, void *interlock)
{
struct rumpcpu *rcpu;
struct cpu_info *ci;
void *old;
bool domigrate;
bool bound = l->l_pflag & LP_BOUND;
l->l_stat = LSRUN;
/*
* First, try fastpath: if we were the previous user of the
* CPU, everything is in order cachewise and we can just
* proceed to use it.
*
* If we are a different thread (i.e. CAS fails), we must go
* through a memory barrier to ensure we get a truthful
* view of the world.
*/
KASSERT(l->l_target_cpu != NULL);
rcpu = cpuinfo_to_rumpcpu(l->l_target_cpu);
if (atomic_cas_ptr(&rcpu->rcpu_prevlwp, l, RCPULWP_BUSY) == l) {
if (interlock == rcpu->rcpu_mtx)
rumpuser_mutex_exit(rcpu->rcpu_mtx);
SCHED_FASTPATH(rcpu);
/* jones, you're the man */
goto fastlane;
}
/*
* Else, it's the slowpath for us. First, determine if we
* can migrate.
*/
if (ncpu == 1)
domigrate = false;
else
domigrate = true;
/* Take lock. This acts as a load barrier too. */
if (interlock != rcpu->rcpu_mtx)
rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
for (;;) {
SCHED_SLOWPATH(rcpu);
old = atomic_swap_ptr(&rcpu->rcpu_prevlwp, RCPULWP_WANTED);
/* CPU is free? */
if (old != RCPULWP_BUSY && old != RCPULWP_WANTED) {
if (atomic_cas_ptr(&rcpu->rcpu_prevlwp,
RCPULWP_WANTED, RCPULWP_BUSY) == RCPULWP_WANTED) {
break;
}
}
/*
* Do we want to migrate once?
* This may need a slightly better algorithm, or we
* might cache pingpong eternally for non-frequent
* threads.
*/
if (domigrate && !bound) {
domigrate = false;
SCHED_MIGRATED(rcpu);
rumpuser_mutex_exit(rcpu->rcpu_mtx);
rcpu = getnextcpu();
rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
continue;
}
/* Want CPU, wait until it's released an retry */
rcpu->rcpu_wanted++;
rumpuser_cv_wait_nowrap(rcpu->rcpu_cv, rcpu->rcpu_mtx);
rcpu->rcpu_wanted--;
}
rumpuser_mutex_exit(rcpu->rcpu_mtx);
fastlane:
ci = rcpu->rcpu_ci;
l->l_cpu = l->l_target_cpu = ci;
l->l_mutex = rcpu->rcpu_ci->ci_schedstate.spc_mutex;
l->l_ncsw++;
l->l_stat = LSONPROC;
/*
* No interrupts, so ci_curlwp === cpu_onproc.
* Okay, we could make an attempt to not set cpu_onproc
* in the case that an interrupt is scheduled immediately
* after a user proc, but leave that for later.
*/
ci->ci_curlwp = ci->ci_data.cpu_onproc = l;
}
void
kern_preprom(void)
{
for (;;) {
/*
* Load the current CPU pointer and examine the mutex_ready bit.
* It doesn't matter if we are preempted here because we are
* only trying to determine if we are in the *set* of mutex
* ready CPUs. We cannot disable preemption until we confirm
* that we are running on a CPU in this set, since a call to
* kpreempt_disable() requires access to curthread.
*/
processorid_t cpuid = getprocessorid();
cpu_t *cp = cpu[cpuid];
cpu_t *prcp;
if (panicstr)
return; /* just return if we are currently panicking */
if (CPU_IN_SET(cpu_ready_set, cpuid) && cp->cpu_m.mutex_ready) {
/*
* Disable premption, and reload the current CPU. We
* can't move from a mutex_ready cpu to a non-ready cpu
* so we don't need to re-check cp->cpu_m.mutex_ready.
*/
kpreempt_disable();
cp = CPU;
ASSERT(cp->cpu_m.mutex_ready);
/*
* Try the lock. If we don't get the lock, re-enable
* preemption and see if we should sleep. If we are
* already the lock holder, remove the effect of the
* previous kpreempt_disable() before returning since
* preemption was disabled by an earlier kern_preprom.
*/
prcp = atomic_cas_ptr((void *)&prom_cpu, NULL, cp);
if (prcp == NULL ||
(prcp == cp && prom_thread == curthread)) {
if (prcp == cp)
kpreempt_enable();
break;
}
kpreempt_enable();
/*
* We have to be very careful here since both prom_cpu
* and prcp->cpu_m.mutex_ready can be changed at any
* time by a non mutex_ready cpu holding the lock.
* If the owner is mutex_ready, holding prom_mutex
* prevents kern_postprom() from completing. If the
* owner isn't mutex_ready, we only know it will clear
* prom_cpu before changing cpu_m.mutex_ready, so we
* issue a membar after checking mutex_ready and then
* re-verify that prom_cpu is still held by the same
* cpu before actually proceeding to cv_wait().
*/
mutex_enter(&prom_mutex);
prcp = prom_cpu;
if (prcp != NULL && prcp->cpu_m.mutex_ready != 0) {
membar_consumer();
if (prcp == prom_cpu)
cv_wait(&prom_cv, &prom_mutex);
}
mutex_exit(&prom_mutex);
} else {
/*
* If we are not yet mutex_ready, just attempt to grab
* the lock. If we get it or already hold it, break.
*/
ASSERT(getpil() == PIL_MAX);
prcp = atomic_cas_ptr((void *)&prom_cpu, NULL, cp);
if (prcp == NULL || prcp == cp)
break;
}
}
/*
* We now hold the prom_cpu lock. Increment the hold count by one
* and assert our current state before returning to the caller.
*/
atomic_inc_32(&prom_holdcnt);
ASSERT(prom_holdcnt >= 1);
prom_thread = curthread;
}
void* atomicCompareExchange(void* volatile& val, void* exch, void* comp)
{
return atomic_cas_ptr(&val, comp, exch);
}
/**
* Set new data on a thread-safe global variable
*
* @param [in] var Pointer to thread-safe global variable
* @param [in] cfdata New value for the thread-safe global variable
* @param [out] new_version New version number
*
* @return 0 on success, or a system error such as ENOMEM.
*/
int
pthread_var_set_np(pthread_var_np_t vp, void *cfdata,
uint64_t *new_version)
{
int err;
size_t i;
struct var *v;
struct vwrapper *old_wrapper = NULL;
struct vwrapper *wrapper;
struct vwrapper *tmp;
uint64_t vers;
uint64_t tmp_version;
uint64_t nref;
if (cfdata == NULL)
return EINVAL;
if (new_version == NULL)
new_version = &vers;
*new_version = 0;
/* Build a wrapper for the new value */
if ((wrapper = calloc(1, sizeof(*wrapper))) == NULL)
return errno;
/*
* The var itself holds a reference to the current value, thus its
* nref starts at 1, but that is made so further below.
*/
wrapper->dtor = vp->dtor;
wrapper->nref = 0;
wrapper->ptr = cfdata;
if ((err = pthread_mutex_lock(&vp->write_lock)) != 0) {
free(wrapper);
return err;
}
/* vp->next_version is stable because we hold the write_lock */
*new_version = wrapper->version = atomic_read_64(&vp->next_version);
/* Grab the next slot */
v = vp->vars[(*new_version + 1) & 0x1].other;
old_wrapper = atomic_read_ptr((volatile void **)&v->wrapper);
if (*new_version == 0) {
/* This is the first write; set wrapper on both slots */
for (i = 0; i < sizeof(vp->vars)/sizeof(vp->vars[0]); i++) {
v = &vp->vars[i];
nref = atomic_inc_32_nv(&wrapper->nref);
v->version = 0;
tmp = atomic_cas_ptr((volatile void **)&v->wrapper,
old_wrapper, wrapper);
assert(tmp == old_wrapper && tmp == NULL);
}
assert(nref > 1);
tmp_version = atomic_inc_64_nv(&vp->next_version);
assert(tmp_version == 1);
/* Signal waiters */
(void) pthread_mutex_lock(&vp->waiter_lock);
(void) pthread_cond_signal(&vp->waiter_cv); /* no thundering herd */
(void) pthread_mutex_unlock(&vp->waiter_lock);
return pthread_mutex_unlock(&vp->write_lock);
}
nref = atomic_inc_32_nv(&wrapper->nref);
assert(nref == 1);
assert(old_wrapper != NULL && old_wrapper->nref > 0);
/* Wait until that slot is quiescent before mutating it */
if ((err = pthread_mutex_lock(&vp->cv_lock)) != 0) {
(void) pthread_mutex_unlock(&vp->write_lock);
free(wrapper);
return err;
}
while (atomic_read_32(&v->nreaders) > 0) {
/*
* We have a separate lock for writing vs. waiting so that no
* other writer can steal our march. All writers will enter,
* all writers will finish. We got here by winning the race for
* the writer lock, so we'll hold onto it, and thus avoid having
* to restart here.
*/
if ((err = pthread_cond_wait(&vp->cv, &vp->cv_lock)) != 0) {
(void) pthread_mutex_unlock(&vp->cv_lock);
(void) pthread_mutex_unlock(&vp->write_lock);
free(wrapper);
return err;
}
}
if ((err = pthread_mutex_unlock(&vp->cv_lock)) != 0) {
(void) pthread_mutex_unlock(&vp->write_lock);
free(wrapper);
return err;
}
/* Update that now quiescent slot; these are the release operations */
tmp = atomic_cas_ptr((volatile void **)&v->wrapper, old_wrapper, wrapper);
assert(tmp == old_wrapper);
v->version = *new_version;
tmp_version = atomic_inc_64_nv(&vp->next_version);
assert(tmp_version == *new_version + 1);
assert(v->version > v->other->version);
/* Release the old cf */
assert(old_wrapper != NULL && old_wrapper->nref > 0);
wrapper_free(old_wrapper);
/* Done */
return pthread_mutex_unlock(&vp->write_lock);
}
/* Lock-less utility to grow the logical slot array */
static int
grow_slots(pthread_var_np_t vp, uint32_t slot_idx, int tries)
{
uint32_t nslots = 0;
uint32_t additions;
uint32_t i;
volatile struct slots **slotsp;
struct slots *new_slots;
for (slotsp = &vp->slots;
atomic_read_ptr((volatile void **)slotsp) != NULL;
slotsp = &((struct slots *)atomic_read_ptr((volatile void **)slotsp))->next)
nslots += (*slotsp)->slot_count;
if (nslots > slot_idx)
return 0;
if (tries < 1)
return EAGAIN; /* shouldn't happen; XXX assert? */
if ((new_slots = calloc(1, sizeof(*new_slots))) == NULL)
return errno;
additions = (nslots == 0) ? 4 : nslots + nslots / 2;
while (slot_idx >= nslots + additions) {
/*
* In this case we're racing with other readers to grow the slot
* list, but if we lose the race then our slot_idx may not be
* covered in the list as grown by the winner. We may have to
* try again.
*
* There's always a winner, so eventually we won't need to try
* again.
*/
additions += additions + additions / 2;
tries++;
}
assert(slot_idx - nslots < additions);
new_slots->slot_array = calloc(additions, sizeof(*new_slots->slot_array));
if (new_slots->slot_array == NULL) {
free(new_slots);
return errno;
}
new_slots->slot_count = additions;
new_slots->slot_base = nslots;
for (i = 0; i < additions; i++) {
new_slots->slot_array[i].in_use = 0;
new_slots->slot_array[i].value = 0;
new_slots->slot_array[i].vp = vp;
}
/* Reserve the slot we wanted (new slots not added yet) */
atomic_write_32(&new_slots->slot_array[slot_idx - nslots].in_use, 1);
/* Add new slots to logical array of slots */
if (atomic_cas_ptr((volatile void **)slotsp, NULL, new_slots) != NULL) {
/*
* We lost the race to grow the array. The index we wanted is
* not guaranteed to be covered by the array as grown by the
* winner. We fall through to recurse to repeat.
*
* See commentary above where tries is incremented.
*/
free(new_slots->slot_array);
free(new_slots);
}
/*
* If we won the race to grow the array then the index we wanted is
* guaranteed to be present and recursing here is cheap. If we lost
* the race we need to retry. We could goto the top of the function
* though, just in case there's no tail call optimization.
*/
return grow_slots(vp, slot_idx, tries - 1);
}
/*
* Add a global or local attribute to a Multidata. Global attribute
* association is specified by a NULL packet descriptor.
*/
pattr_t *
mmd_addpattr(multidata_t *mmd, pdesc_t *pd, pattrinfo_t *pai,
boolean_t persistent, int kmflags)
{
patbkt_t **tbl_p;
patbkt_t *tbl, *o_tbl;
patbkt_t *bkt;
pattr_t *pa;
uint_t size;
ASSERT(mmd != NULL);
ASSERT(mmd->mmd_magic == MULTIDATA_MAGIC);
ASSERT(pd == NULL || pd->pd_magic == PDESC_MAGIC);
ASSERT(pai != NULL);
/* pointer to the attribute hash table (local or global) */
tbl_p = pd != NULL ? &(pd->pd_pattbl) : &(mmd->mmd_pattbl);
/*
* See if the hash table has not yet been created; if so,
* we create the table and store its address atomically.
*/
if ((tbl = *tbl_p) == NULL) {
tbl = kmem_cache_alloc(pattbl_cache, kmflags);
if (tbl == NULL)
return (NULL);
/* if someone got there first, use his table instead */
if ((o_tbl = atomic_cas_ptr(tbl_p, NULL, tbl)) != NULL) {
kmem_cache_free(pattbl_cache, tbl);
tbl = o_tbl;
}
}
ASSERT(tbl->pbkt_tbl_sz > 0);
bkt = &(tbl[PATTBL_HASH(pai->type, tbl->pbkt_tbl_sz)]);
/* attribute of the same type already exists? */
if ((pa = mmd_find_pattr(bkt, pai->type)) != NULL)
return (NULL);
size = sizeof (*pa) + pai->len;
if ((pa = kmem_zalloc(size, kmflags)) == NULL)
return (NULL);
pa->pat_magic = PATTR_MAGIC;
pa->pat_lock = &(bkt->pbkt_lock);
pa->pat_mmd = mmd;
pa->pat_buflen = size;
pa->pat_type = pai->type;
pai->buf = pai->len > 0 ? ((uchar_t *)(pa + 1)) : NULL;
if (persistent)
pa->pat_flags = PATTR_PERSIST;
/* insert attribute at end of hash chain */
mutex_enter(&(bkt->pbkt_lock));
insque(&(pa->pat_next), bkt->pbkt_pattr_q.ql_prev);
mutex_exit(&(bkt->pbkt_lock));
return (pa);
}