static VALUE ir_compare_and_set(volatile VALUE self, VALUE expect_value, VALUE new_value) {
#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050
if (OSAtomicCompareAndSwap64(expect_value, new_value, &DATA_PTR(self))) {
return Qtrue;
}
#elif defined(__sun)
/* Assuming VALUE is uintptr_t */
/* Based on the definition of uintptr_t from /usr/include/sys/int_types.h */
#if defined(_LP64) || defined(_I32LPx)
/* 64-bit: uintptr_t === unsigned long */
if (atomic_cas_ulong((uintptr_t *) &DATA_PTR(self), expect_value, new_value)) {
return Qtrue;
}
#else
/* 32-bit: uintptr_t === unsigned int */
if (atomic_cas_uint((uintptr_t *) &DATA_PTR(self), expect_value, new_value)) {
return Qtrue;
}
#endif
#elif defined _MSC_VER && defined _M_AMD64
if (InterlockedCompareExchange64((LONGLONG*)&DATA_PTR(self), new_value, expect_value)) {
return Qtrue;
}
#elif defined _MSC_VER && defined _M_IX86
if (InterlockedCompareExchange((LONG*)&DATA_PTR(self), new_value, expect_value)) {
return Qtrue;
}
#else
if (__sync_bool_compare_and_swap(&DATA_PTR(self), expect_value, new_value)) {
return Qtrue;
}
#endif
return Qfalse;
}
int atomicCompareExchange(volatile atomic_t& val, int exch, int comp)
{
volatile ulong_t* uvalue = reinterpret_cast<volatile ulong_t*>(&val.l);
volatile ulong_t uex = exch;
volatile ulong_t ucmp = comp;
return atomic_cas_ulong(uvalue, ucmp, uex);
}
/*ARGSUSED*/
void *
segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
{
size_t size;
kthread_t *t = curthread;
segkmem_lpcb_t *lpcb = &segkmem_lpcb;
ASSERT(sizep != NULL);
size = *sizep;
if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
!(vmflag & SEGKMEM_SHARELOCKED)) {
size_t kmemlp_qnt = segkmem_kmemlp_quantum;
size_t asize = P2ROUNDUP(size, kmemlp_qnt);
void *addr = NULL;
ulong_t *lpthrtp = &lpcb->lp_throttle;
ulong_t lpthrt = *lpthrtp;
int dowakeup = 0;
int doalloc = 1;
ASSERT(kmem_lp_arena != NULL);
ASSERT(asize >= size);
if (lpthrt != 0) {
/* try to update the throttle value */
lpthrt = atomic_inc_ulong_nv(lpthrtp);
if (lpthrt >= segkmem_lpthrottle_max) {
lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
segkmem_lpthrottle_max / 4);
}
/*
* when we get above throttle start do an exponential
* backoff at trying large pages and reaping
*/
if (lpthrt > segkmem_lpthrottle_start &&
!ISP2(lpthrt)) {
lpcb->allocs_throttled++;
lpthrt--;
if (ISP2(lpthrt))
kmem_reap();
return (segkmem_alloc(vmp, size, vmflag));
}
}
if (!(vmflag & VM_NOSLEEP) &&
segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
/*
* we are low on free memory in kmem_lp_arena
* we let only one guy to allocate heap_lp
* quantum size chunk that everybody is going to
* share
*/
mutex_enter(&lpcb->lp_lock);
if (lpcb->lp_wait) {
/* we are not the first one - wait */
cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
if (vmem_size(kmem_lp_arena, VMEM_FREE) <
kmemlp_qnt) {
doalloc = 0;
}
} else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
kmemlp_qnt) {
/*
* we are the first one, make sure we import
* a large page
*/
if (asize == kmemlp_qnt)
asize += kmemlp_qnt;
dowakeup = 1;
lpcb->lp_wait = 1;
}
mutex_exit(&lpcb->lp_lock);
}
/*
* VM_ABORT flag prevents sleeps in vmem_xalloc when
* large pages are not available. In that case this allocation
* attempt will fail and we will retry allocation with small
* pages. We also do not want to panic if this allocation fails
* because we are going to retry.
*/
if (doalloc) {
addr = vmem_alloc(kmem_lp_arena, asize,
(vmflag | VM_ABORT) & ~VM_PANIC);
if (dowakeup) {
mutex_enter(&lpcb->lp_lock);
ASSERT(lpcb->lp_wait != 0);
lpcb->lp_wait = 0;
cv_broadcast(&lpcb->lp_cv);
mutex_exit(&lpcb->lp_lock);
}
}
if (addr != NULL) {
*sizep = asize;
*lpthrtp = 0;
return (addr);
}
if (vmflag & VM_NOSLEEP)
lpcb->nosleep_allocs_failed++;
else
lpcb->sleep_allocs_failed++;
lpcb->alloc_bytes_failed += size;
/* if large page throttling is not started yet do it */
if (segkmem_use_lpthrottle && lpthrt == 0) {
lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
}
}
return (segkmem_alloc(vmp, size, vmflag));
}
void
clock_tick_schedule(int one_sec)
{
ulong_t active;
int i, end;
clock_tick_set_t *csp;
cpu_t *cp;
if (clock_cpu_id != CPU->cpu_id)
clock_cpu_id = CPU->cpu_id;
if (clock_tick_single_threaded) {
/*
* Each tick cycle, start the scan from a different
* CPU for the sake of fairness.
*/
end = clock_tick_total_cpus;
clock_tick_scan++;
if (clock_tick_scan >= end)
clock_tick_scan = 0;
clock_tick_execute_common(0, clock_tick_scan, end,
LBOLT_NO_ACCOUNT, 1);
return;
}
/*
* If the previous invocation of handlers is not yet finished, then
* simply increment a pending count and return. Eventually when they
* finish, the pending count is passed down to the next set of
* handlers to process. This way, ticks that have already elapsed
* in the past are handled as quickly as possible to minimize the
* chances of threads getting away before their pending ticks are
* accounted. The other benefit is that if the pending count is
* more than one, it can be handled by a single invocation of
* clock_tick(). This is a good optimization for large configuration
* busy systems where tick accounting can get backed up for various
* reasons.
*/
clock_tick_pending++;
active = clock_tick_active;
active = atomic_cas_ulong(&clock_tick_active, active, active);
if (active)
return;
/*
* We want to handle the clock CPU here. If we
* scheduled the accounting for the clock CPU to another
* processor, that processor will find only the clock() thread
* running and not account for any user thread below it. Also,
* we want to handle this before we block on anything and allow
* the pinned thread below the current thread to escape.
*/
clock_tick_process(CPU, LBOLT_NO_ACCOUNT, clock_tick_pending);
mutex_enter(&clock_tick_lock);
/*
* Schedule each set on a separate processor.
*/
cp = clock_cpu_list;
for (i = 0; i < clock_tick_nsets; i++) {
csp = &clock_tick_set[i];
/*
* Pick the next online CPU in list for scheduling tick
* accounting. The clock_tick_lock is held by the caller.
* So, CPU online/offline cannot muck with this while
* we are picking our CPU to X-call.
*/
if (cp == CPU)
cp = cp->cpu_next_onln;
/*
* Each tick cycle, start the scan from a different
* CPU for the sake of fairness.
*/
csp->ct_scan++;
if (csp->ct_scan >= csp->ct_end)
csp->ct_scan = csp->ct_start;
clock_tick_schedule_one(csp, clock_tick_pending, cp->cpu_id);
cp = cp->cpu_next_onln;
}
if (one_sec) {
/*
* Move the CPU pointer around every second. This is so
* all the CPUs can be X-called in a round-robin fashion
* to evenly distribute the X-calls. We don't do this
* at a faster rate than this because we don't want
* to affect cache performance negatively.
*/
clock_cpu_list = clock_cpu_list->cpu_next_onln;
}
mutex_exit(&clock_tick_lock);
clock_tick_pending = 0;
}
