/*
* Hunt down any pages in the system that have not yet been retired, invoking
* the provided callback function on each of them.
*/
void
page_retire_hunt(void (*callback)(page_t *))
{
page_t *pp;
page_t *first;
uint64_t tbr, found;
int i;
PR_DEBUG(prd_hunt);
if (PR_KSTAT_PENDING == 0) {
return;
}
PR_DEBUG(prd_dohunt);
found = 0;
mutex_enter(&pr_q_mutex);
tbr = PR_KSTAT_PENDING;
for (i = 0; i < PR_PENDING_QMAX; i++) {
if ((pp = pr_pending_q[i]) != NULL) {
mutex_exit(&pr_q_mutex);
callback(pp);
mutex_enter(&pr_q_mutex);
found++;
}
}
if (PR_KSTAT_EQFAIL == PR_KSTAT_DQFAIL && found == tbr) {
mutex_exit(&pr_q_mutex);
PR_DEBUG(prd_earlyhunt);
return;
}
mutex_exit(&pr_q_mutex);
PR_DEBUG(prd_latehunt);
/*
* We've lost track of a page somewhere. Hunt it down.
*/
memsegs_lock(0);
pp = first = page_first();
do {
if (PP_PR_REQ(pp)) {
callback(pp);
if (++found == tbr) {
break; /* got 'em all */
}
}
} while ((pp = page_next(pp)) != first);
memsegs_unlock(0);
}
/*
* Page retire self-test. For now, it always returns 0.
*/
int
page_retire_test(void)
{
page_t *first, *pp, *cpp, *cpp2, *lpp;
/*
* Tests the corner case where a large page can't be retired
* because one of the constituent pages is locked. We mark
* one page to be retired and try to retire it, and mark the
* other page to be retired but don't try to retire it, so
* that page_unlock() in the failure path will recurse and try
* to retire THAT page. This is the worst possible situation
* we can get ourselves into.
*/
memsegs_lock(0);
pp = first = page_first();
do {
if (pp->p_szc && PP_PAGEROOT(pp) == pp) {
cpp = pp + 1;
lpp = PP_ISFREE(pp)? pp : pp + 2;
cpp2 = pp + 3;
if (!page_trylock(lpp, pp == lpp? SE_EXCL : SE_SHARED))
continue;
if (!page_trylock(cpp, SE_EXCL)) {
page_unlock(lpp);
continue;
}
page_settoxic(cpp, PR_FMA | PR_BUSY);
page_settoxic(cpp2, PR_FMA);
page_tryretire(cpp); /* will fail */
page_unlock(lpp);
(void) page_retire(cpp->p_pagenum, PR_FMA);
(void) page_retire(cpp2->p_pagenum, PR_FMA);
}
} while ((pp = page_next(pp)) != first);
memsegs_unlock(0);
return (0);
}
/*
* We want to add memory, but have no spare page_t structures. Use some of
* our new memory for the page_t structures.
*
* Somewhat similar to kphysm_add_memory_dynamic(), but simpler.
*/
static int
balloon_init_new_pages(mfn_t framelist[], pgcnt_t count)
{
pgcnt_t metapgs, totalpgs, num_pages;
paddr_t metasz;
pfn_t meta_start;
page_t *page_array;
caddr_t va;
int i, rv, locked;
mem_structs_t *mem;
struct memseg *segp;
/* Calculate the number of pages we're going to add */
totalpgs = bln_stats.bln_new_target - bln_stats.bln_current_pages;
/*
* The following calculates the number of "meta" pages -- the pages
* that will be required to hold page_t structures for all new pages.
* Proof of this calculation is left up to the reader.
*/
metapgs = totalpgs - (((uint64_t)(totalpgs) << PAGESHIFT) /
(PAGESIZE + sizeof (page_t)));
/*
* Given the number of page_t structures we need, is there also
* room in our meta pages for a memseg and memlist struct?
* If not, we'll need one more meta page.
*/
if ((metapgs << PAGESHIFT) < (totalpgs * sizeof (page_t) +
MEM_STRUCT_SIZE))
metapgs++;
/*
* metapgs is calculated from totalpgs, which may be much larger than
* count. If we don't have enough pages, all of the pages in this
* batch will be made meta pages, and a future trip through
* balloon_inc_reservation() will add the rest of the meta pages.
*/
if (metapgs > count)
metapgs = count;
/*
* Figure out the number of page_t structures that can fit in metapgs
*
* This will cause us to initialize more page_t structures than we
* need - these may be used in future memory increases.
*/
metasz = pfn_to_pa(metapgs);
num_pages = (metasz - MEM_STRUCT_SIZE) / sizeof (page_t);
DTRACE_PROBE3(balloon__alloc__stats, pgcnt_t, totalpgs, pgcnt_t,
num_pages, pgcnt_t, metapgs);
/*
* We only increment mfn_count by count, not num_pages, to keep the
* space of all valid pfns contiguous. This means we create page_t
* structures with invalid pagenums -- we deal with this situation
* in balloon_page_sub.
*/
mfn_count += count;
/*
* Get a VA for the pages that will hold page_t and other structures.
* The memseg and memlist structures will go at the beginning, with
* the page_t structures following.
*/
va = (caddr_t)vmem_alloc(heap_arena, metasz, VM_SLEEP);
/* LINTED: improper alignment */
mem = (mem_structs_t *)va;
page_array = mem->pages;
meta_start = bln_stats.bln_max_pages;
/*
* Set the mfn to pfn mapping for the meta pages.
*/
locked = balloon_lock_contig_pfnlist(metapgs);
for (i = 0; i < metapgs; i++) {
reassign_pfn(bln_stats.bln_max_pages + i, framelist[i]);
}
if (locked)
unlock_contig_pfnlist();
/*
* For our meta pages, map them in and zero the page.
* This will be the first time touching the new pages.
*/
hat_devload(kas.a_hat, va, metasz, bln_stats.bln_max_pages,
PROT_READ | PROT_WRITE,
HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
bzero(va, metasz);
/*
* Initialize the page array for the new pages.
*/
for (i = 0; i < metapgs; i++) {
page_array[i].p_pagenum = bln_stats.bln_max_pages++;
page_array[i].p_offset = (u_offset_t)-1;
page_iolock_init(&page_array[i]);
rv = page_lock(&page_array[i], SE_EXCL, NULL, P_NO_RECLAIM);
ASSERT(rv == 1);
}
/*
* For the rest of the pages, initialize the page_t struct and
* add them to the free list
*/
for (i = metapgs; i < num_pages; i++) {
page_array[i].p_pagenum = bln_stats.bln_max_pages++;
page_array[i].p_offset = (u_offset_t)-1;
page_iolock_init(&page_array[i]);
rv = page_lock(&page_array[i], SE_EXCL, NULL, P_NO_RECLAIM);
ASSERT(rv == 1);
balloon_page_add(&page_array[i]);
}
/*
* Remember where I said that we don't call this function? The missing
* code right here is why. We need to set up kpm mappings for any new
* pages coming in. However, if someone starts up a domain with small
* memory, then greatly increases it, we could get in some horrible
* deadlock situations as we steal page tables for kpm use, and
* userland applications take them right back before we can use them
* to set up our new memory. Once a way around that is found, and a
* few other changes are made, we'll be able to enable this code.
*/
/*
* Update kernel structures, part 1: memsegs list
*/
mem->memseg.pages_base = meta_start;
mem->memseg.pages_end = bln_stats.bln_max_pages - 1;
mem->memseg.pages = &page_array[0];
mem->memseg.epages = &page_array[num_pages - 1];
mem->memseg.next = NULL;
memsegs_lock(1);
for (segp = memsegs; segp->next != NULL; segp = segp->next)
;
segp->next = &mem->memseg;
memsegs_unlock(1);
/*
* Update kernel structures, part 2: mem_node array
*/
mem_node_add_slice(meta_start, bln_stats.bln_max_pages);
/*
* Update kernel structures, part 3: phys_install array
* (*sigh* how many of these things do we need?)
*/
memlist_write_lock();
memlist_add(pfn_to_pa(meta_start), num_pages, &mem->memlist,
&phys_install);
memlist_write_unlock();
build_pfn_hash();
return (metapgs);
}
