/*
* The page_retire_thread loops forever, looking to see if there are
* pages still waiting to be retired.
*/
static void
page_retire_thread(void)
{
callb_cpr_t c;
CALLB_CPR_INIT(&c, &pr_thread_mutex, callb_generic_cpr, "page_retire");
mutex_enter(&pr_thread_mutex);
for (;;) {
if (pr_enable && PR_KSTAT_PENDING) {
/*
* Sigh. It's SO broken how we have to try to shake
* loose the holder of the page. Since we have no
* idea who or what has it locked, we go bang on
* every door in the city to try to locate it.
*/
kmem_reap();
seg_preap();
page_retire_hunt(page_retire_thread_cb);
CALLB_CPR_SAFE_BEGIN(&c);
(void) cv_timedwait(&pr_cv, &pr_thread_mutex,
lbolt + pr_thread_shortwait);
CALLB_CPR_SAFE_END(&c, &pr_thread_mutex);
} else {
CALLB_CPR_SAFE_BEGIN(&c);
(void) cv_timedwait(&pr_cv, &pr_thread_mutex,
lbolt + pr_thread_longwait);
CALLB_CPR_SAFE_END(&c, &pr_thread_mutex);
}
}
/*NOTREACHED*/
}
/*ARGSUSED*/
void *
segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
{
size_t size;
kthread_t *t = curthread;
segkmem_lpcb_t *lpcb = &segkmem_lpcb;
ASSERT(sizep != NULL);
size = *sizep;
if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
!(vmflag & SEGKMEM_SHARELOCKED)) {
size_t kmemlp_qnt = segkmem_kmemlp_quantum;
size_t asize = P2ROUNDUP(size, kmemlp_qnt);
void *addr = NULL;
ulong_t *lpthrtp = &lpcb->lp_throttle;
ulong_t lpthrt = *lpthrtp;
int dowakeup = 0;
int doalloc = 1;
ASSERT(kmem_lp_arena != NULL);
ASSERT(asize >= size);
if (lpthrt != 0) {
/* try to update the throttle value */
lpthrt = atomic_inc_ulong_nv(lpthrtp);
if (lpthrt >= segkmem_lpthrottle_max) {
lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
segkmem_lpthrottle_max / 4);
}
/*
* when we get above throttle start do an exponential
* backoff at trying large pages and reaping
*/
if (lpthrt > segkmem_lpthrottle_start &&
!ISP2(lpthrt)) {
lpcb->allocs_throttled++;
lpthrt--;
if (ISP2(lpthrt))
kmem_reap();
return (segkmem_alloc(vmp, size, vmflag));
}
}
if (!(vmflag & VM_NOSLEEP) &&
segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
/*
* we are low on free memory in kmem_lp_arena
* we let only one guy to allocate heap_lp
* quantum size chunk that everybody is going to
* share
*/
mutex_enter(&lpcb->lp_lock);
if (lpcb->lp_wait) {
/* we are not the first one - wait */
cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
if (vmem_size(kmem_lp_arena, VMEM_FREE) <
kmemlp_qnt) {
doalloc = 0;
}
} else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
kmemlp_qnt) {
/*
* we are the first one, make sure we import
* a large page
*/
if (asize == kmemlp_qnt)
asize += kmemlp_qnt;
dowakeup = 1;
lpcb->lp_wait = 1;
}
mutex_exit(&lpcb->lp_lock);
}
/*
* VM_ABORT flag prevents sleeps in vmem_xalloc when
* large pages are not available. In that case this allocation
* attempt will fail and we will retry allocation with small
* pages. We also do not want to panic if this allocation fails
* because we are going to retry.
*/
if (doalloc) {
addr = vmem_alloc(kmem_lp_arena, asize,
(vmflag | VM_ABORT) & ~VM_PANIC);
if (dowakeup) {
mutex_enter(&lpcb->lp_lock);
ASSERT(lpcb->lp_wait != 0);
lpcb->lp_wait = 0;
cv_broadcast(&lpcb->lp_cv);
mutex_exit(&lpcb->lp_lock);
}
}
if (addr != NULL) {
*sizep = asize;
*lpthrtp = 0;
return (addr);
}
if (vmflag & VM_NOSLEEP)
lpcb->nosleep_allocs_failed++;
else
lpcb->sleep_allocs_failed++;
lpcb->alloc_bytes_failed += size;
/* if large page throttling is not started yet do it */
if (segkmem_use_lpthrottle && lpthrt == 0) {
lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
}
}
return (segkmem_alloc(vmp, size, vmflag));
}
/*
* This function is called when we want to decrease the memory reservation
* of our domain. Allocate the memory and make a hypervisor call to give
* it back.
*/
static spgcnt_t
balloon_dec_reservation(ulong_t debit)
{
int i, locked;
long rv;
ulong_t request;
page_t *pp;
bzero(mfn_frames, sizeof (mfn_frames));
bzero(pfn_frames, sizeof (pfn_frames));
if (debit > FRAME_ARRAY_SIZE) {
debit = FRAME_ARRAY_SIZE;
}
request = debit;
/*
* Don't bother if there isn't a safe amount of kmem left.
*/
if (kmem_avail() < balloon_minkmem) {
kmem_reap();
if (kmem_avail() < balloon_minkmem)
return (0);
}
if (page_resv(request, KM_NOSLEEP) == 0) {
return (0);
}
xen_block_migrate();
for (i = 0; i < debit; i++) {
pp = page_get_high_mfn(new_high_mfn);
new_high_mfn = 0;
if (pp == NULL) {
/*
* Call kmem_reap(), then try once more,
* but only if there is a safe amount of
* kmem left.
*/
kmem_reap();
if (kmem_avail() < balloon_minkmem ||
(pp = page_get_high_mfn(0)) == NULL) {
debit = i;
break;
}
}
ASSERT(PAGE_EXCL(pp));
ASSERT(!hat_page_is_mapped(pp));
balloon_page_add(pp);
pfn_frames[i] = pp->p_pagenum;
mfn_frames[i] = pfn_to_mfn(pp->p_pagenum);
}
if (debit == 0) {
xen_allow_migrate();
page_unresv(request);
return (0);
}
/*
* We zero all the pages before we start reassigning them in order to
* minimize the time spent holding the lock on the contig pfn list.
*/
if (balloon_zero_memory) {
for (i = 0; i < debit; i++) {
pfnzero(pfn_frames[i], 0, PAGESIZE);
}
}
/*
* Remove all mappings for the pfns from the system
*/
locked = balloon_lock_contig_pfnlist(debit);
for (i = 0; i < debit; i++) {
reassign_pfn(pfn_frames[i], MFN_INVALID);
}
if (locked)
unlock_contig_pfnlist();
rv = balloon_free_pages(debit, mfn_frames, NULL, NULL);
if (rv < 0) {
cmn_err(CE_WARN, "Attempt to return pages to the hypervisor "
"failed - up to %lu pages lost (error = %ld)", debit, rv);
rv = 0;
} else if (rv != debit) {
panic("Unexpected return value (%ld) from decrease reservation "
"hypervisor call", rv);
}
xen_allow_migrate();
if (debit != request)
page_unresv(request - debit);
return (rv);
}
/*
* Schedule rate for paging.
* Rate is linear interpolation between
* slowscan with lotsfree and fastscan when out of memory.
*/
static void
schedpaging(void *arg)
{
spgcnt_t vavail;
if (freemem < lotsfree + needfree + kmem_reapahead)
kmem_reap();
if (freemem < lotsfree + needfree + seg_preapahead)
seg_preap();
if (kcage_on && (kcage_freemem < kcage_desfree || kcage_needfree))
kcage_cageout_wakeup();
if (mutex_tryenter(&pageout_mutex)) {
/* pageout() not running */
nscan = 0;
vavail = freemem - deficit;
if (vavail < 0)
vavail = 0;
if (vavail > lotsfree)
vavail = lotsfree;
/*
* Fix for 1161438 (CRS SPR# 73922). All variables
* in the original calculation for desscan were 32 bit signed
* ints. As freemem approaches 0x0 on a system with 1 Gig or
* more of memory, the calculation can overflow. When this
* happens, desscan becomes negative and pageout_scanner()
* stops paging out.
*/
if (needfree) {
desscan = fastscan / RATETOSCHEDPAGING;
} else {
spgcnt_t faststmp, slowstmp, result;
slowstmp = slowscan * vavail;
faststmp = fastscan * (lotsfree - vavail);
result = (slowstmp + faststmp) /
nz(lotsfree) / RATETOSCHEDPAGING;
desscan = (pgcnt_t)result;
}
pageout_ticks = min_pageout_ticks + (lotsfree - vavail) *
(max_pageout_ticks - min_pageout_ticks) / nz(lotsfree);
if (freemem < lotsfree + needfree ||
pageout_sample_cnt < pageout_sample_lim) {
TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
"pageout_cv_signal:freemem %ld", freemem);
cv_signal(&proc_pageout->p_cv);
} else {
/*
* There are enough free pages, no need to
* kick the scanner thread. And next time
* around, keep more of the `highly shared'
* pages.
*/
cv_signal_pageout();
if (po_share > MIN_PO_SHARE) {
po_share >>= 1;
}
}