/*
* Scan page_t's and issue I/O's for modified pages.
*
* Also coalesces consecutive small sized free pages into the next larger
* pagesize. This costs a tiny bit of time in fsflush, but will reduce time
* spent scanning on later passes and for anybody allocating large pages.
*/
static void
fsflush_do_pages()
{
vnode_t *vp;
ulong_t pcount;
hrtime_t timer = gethrtime();
ulong_t releases = 0;
ulong_t nexamined = 0;
ulong_t nlocked = 0;
ulong_t nmodified = 0;
ulong_t ncoalesce = 0;
ulong_t cnt;
int mod;
int fspage = 1;
u_offset_t offset;
uint_t szc;
page_t *coal_page = NULL; /* 1st page in group to coalesce */
uint_t coal_szc = 0; /* size code, coal_page->p_szc */
uint_t coal_cnt = 0; /* count of pages seen */
static ulong_t nscan = 0;
static pgcnt_t last_total_pages = 0;
static page_t *pp = NULL;
/*
* Check to see if total_pages has changed.
*/
if (total_pages != last_total_pages) {
last_total_pages = total_pages;
nscan = (last_total_pages * (tune.t_fsflushr))/v.v_autoup;
}
if (pp == NULL)
pp = memsegs->pages;
pcount = 0;
while (pcount < nscan) {
/*
* move to the next page, skipping over large pages
* and issuing prefetches.
*/
if (pp->p_szc && fspage == 0) {
pfn_t pfn;
pfn = page_pptonum(pp);
cnt = page_get_pagecnt(pp->p_szc);
cnt -= pfn & (cnt - 1);
} else
cnt = 1;
pp = page_nextn(pp, cnt);
prefetch_page_r((void *)pp);
ASSERT(pp != NULL);
pcount += cnt;
/*
* Do a bunch of dirty tests (ie. no locking) to determine
* if we can quickly skip this page. These tests are repeated
* after acquiring the page lock.
*/
++nexamined;
if (PP_ISSWAP(pp)) {
fspage = 0;
coal_page = NULL;
continue;
}
/*
* skip free pages too, but try coalescing them into larger
* pagesizes
*/
if (PP_ISFREE(pp)) {
/*
* skip pages with a file system identity or that
* are already maximum size
*/
fspage = 0;
szc = pp->p_szc;
if (pp->p_vnode != NULL || szc == fsf_npgsz - 1) {
coal_page = NULL;
continue;
}
/*
* If not in a coalescing candidate page or the size
* codes are different, start a new candidate.
*/
if (coal_page == NULL || coal_szc != szc) {
/*
* page must be properly aligned
*/
if ((page_pptonum(pp) & fsf_mask[szc]) != 0) {
coal_page = NULL;
continue;
}
coal_page = pp;
coal_szc = szc;
coal_cnt = 1;
continue;
}
/*
* acceptable to add this to existing candidate page
*/
++coal_cnt;
if (coal_cnt < fsf_pgcnt[coal_szc])
continue;
/*
* We've got enough pages to coalesce, so do it.
* After promoting, we clear coal_page, so it will
* take another pass to promote this to an even
* larger page.
*/
++ncoalesce;
(void) page_promote_size(coal_page, coal_szc);
coal_page = NULL;
continue;
} else {
coal_page = NULL;
}
if (PP_ISKAS(pp) ||
PAGE_LOCKED(pp) ||
pp->p_lckcnt != 0 ||
pp->p_cowcnt != 0) {
fspage = 0;
continue;
}
/*
* Reject pages that can't be "exclusively" locked.
*/
if (!page_trylock(pp, SE_EXCL))
continue;
++nlocked;
/*
* After locking the page, redo the above checks.
* Since we locked the page, leave out the PAGE_LOCKED() test.
*/
vp = pp->p_vnode;
if (PP_ISSWAP(pp) ||
PP_ISFREE(pp) ||
vp == NULL ||
PP_ISKAS(pp) ||
(vp->v_flag & VISSWAP) != 0) {
page_unlock(pp);
fspage = 0;
continue;
}
if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
page_unlock(pp);
continue;
}
fspage = 1;
ASSERT(vp->v_type != VCHR);
/*
* Check the modified bit. Leaving the bit alone in hardware.
* It will be cleared if we do the putpage.
*/
if (IS_VMODSORT(vp))
mod = hat_ismod(pp);
else
mod = hat_pagesync(pp,
HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD;
if (mod) {
++nmodified;
offset = pp->p_offset;
/*
* Hold the vnode before releasing the page lock
* to prevent it from being freed and re-used by
* some other thread.
*/
VN_HOLD(vp);
page_unlock(pp);
(void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_ASYNC,
kcred, NULL);
VN_RELE(vp);
} else {
/*
* Catch any pages which should be on the cache list,
* but aren't yet.
*/
if (hat_page_is_mapped(pp) == 0) {
++releases;
(void) page_release(pp, 1);
} else {
page_unlock(pp);
}
}
}
/*
* maintain statistics
* reset every million wakeups, just to avoid overflow
*/
if (++fsf_cycles == 1000000) {
fsf_cycles = 0;
fsf_total.fsf_scan = 0;
fsf_total.fsf_examined = 0;
fsf_total.fsf_locked = 0;
fsf_total.fsf_modified = 0;
fsf_total.fsf_coalesce = 0;
fsf_total.fsf_time = 0;
fsf_total.fsf_releases = 0;
} else {
fsf_total.fsf_scan += fsf_recent.fsf_scan = nscan;
fsf_total.fsf_examined += fsf_recent.fsf_examined = nexamined;
fsf_total.fsf_locked += fsf_recent.fsf_locked = nlocked;
fsf_total.fsf_modified += fsf_recent.fsf_modified = nmodified;
fsf_total.fsf_coalesce += fsf_recent.fsf_coalesce = ncoalesce;
fsf_total.fsf_time += fsf_recent.fsf_time = gethrtime() - timer;
fsf_total.fsf_releases += fsf_recent.fsf_releases = releases;
}
}
/*
* Find, take and return a mutex held by hat_page_demote().
* Called by page_demote_vp_pages() before hat_page_demote() call and by
* routines that want to block hat_page_demote() but can't do it
* via locking all constituent pages.
*
* Return NULL if p_szc is 0.
*
* It should only be used for pages that can be demoted by hat_page_demote()
* i.e. non swapfs file system pages. The logic here is lifted from
* sfmmu_mlspl_enter() except there's no need to worry about p_szc increase
* since the page is locked and not free.
*
* Hash of the root page is used to find the lock.
* To find the root in the presense of hat_page_demote() chageing the location
* of the root this routine relies on the fact that hat_page_demote() changes
* root last.
*
* If NULL is returned pp's p_szc is guaranteed to be 0. If non NULL is
* returned pp's p_szc may be any value.
*/
kmutex_t *
page_szc_lock(page_t *pp)
{
kmutex_t *mtx;
page_t *rootpp;
uint_t szc;
uint_t rszc;
uint_t pszc = pp->p_szc;
ASSERT(pp != NULL);
ASSERT(PAGE_LOCKED(pp));
ASSERT(!PP_ISFREE(pp));
ASSERT(pp->p_vnode != NULL);
ASSERT(!IS_SWAPFSVP(pp->p_vnode));
ASSERT(!PP_ISKAS(pp));
again:
if (pszc == 0) {
VM_STAT_ADD(pszclck_stat[0]);
return (NULL);
}
/* The lock lives in the root page */
rootpp = PP_GROUPLEADER(pp, pszc);
mtx = PAGE_SZC_MUTEX(rootpp);
mutex_enter(mtx);
/*
* since p_szc can only decrease if pp == rootpp
* rootpp will be always the same i.e we have the right root
* regardless of rootpp->p_szc.
* If location of pp's root didn't change after we took
* the lock we have the right root. return mutex hashed off it.
*/
if (pp == rootpp || (rszc = rootpp->p_szc) == pszc) {
VM_STAT_ADD(pszclck_stat[1]);
return (mtx);
}
/*
* root location changed because page got demoted.
* locate the new root.
*/
if (rszc < pszc) {
szc = pp->p_szc;
ASSERT(szc < pszc);
mutex_exit(mtx);
pszc = szc;
VM_STAT_ADD(pszclck_stat[2]);
goto again;
}
VM_STAT_ADD(pszclck_stat[3]);
/*
* current hat_page_demote not done yet.
* wait for it to finish.
*/
mutex_exit(mtx);
rootpp = PP_GROUPLEADER(rootpp, rszc);
mtx = PAGE_SZC_MUTEX(rootpp);
mutex_enter(mtx);
mutex_exit(mtx);
ASSERT(rootpp->p_szc < rszc);
goto again;
}
