PFN_NUMBER
NTAPI
MiFindContiguousPages(IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN PFN_NUMBER SizeInPages,
IN MEMORY_CACHING_TYPE CacheType)
{
PFN_NUMBER Page, PageCount, LastPage, Length, BoundaryMask;
ULONG i = 0;
PMMPFN Pfn1, EndPfn;
KIRQL OldIrql;
PAGED_CODE();
ASSERT(SizeInPages != 0);
//
// Convert the boundary PFN into an alignment mask
//
BoundaryMask = ~(BoundaryPfn - 1);
/* Disable APCs */
KeEnterGuardedRegion();
//
// Loop all the physical memory blocks
//
do
{
//
// Capture the base page and length of this memory block
//
Page = MmPhysicalMemoryBlock->Run[i].BasePage;
PageCount = MmPhysicalMemoryBlock->Run[i].PageCount;
//
// Check how far this memory block will go
//
LastPage = Page + PageCount;
//
// Trim it down to only the PFNs we're actually interested in
//
if ((LastPage - 1) > HighestPfn) LastPage = HighestPfn + 1;
if (Page < LowestPfn) Page = LowestPfn;
//
// Skip this run if it's empty or fails to contain all the pages we need
//
if (!(PageCount) || ((Page + SizeInPages) > LastPage)) continue;
//
// Now scan all the relevant PFNs in this run
//
Length = 0;
for (Pfn1 = MI_PFN_ELEMENT(Page); Page < LastPage; Page++, Pfn1++)
{
//
// If this PFN is in use, ignore it
//
if (MiIsPfnInUse(Pfn1))
{
Length = 0;
continue;
}
//
// If we haven't chosen a start PFN yet and the caller specified an
// alignment, make sure the page matches the alignment restriction
//
if ((!(Length) && (BoundaryPfn)) &&
(((Page ^ (Page + SizeInPages - 1)) & BoundaryMask)))
{
//
// It does not, so bail out
//
continue;
}
//
// Increase the number of valid pages, and check if we have enough
//
if (++Length == SizeInPages)
{
//
// It appears we've amassed enough legitimate pages, rollback
//
Pfn1 -= (Length - 1);
Page -= (Length - 1);
//
// Acquire the PFN lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// Things might've changed for us. Is the page still free?
//
if (MiIsPfnInUse(Pfn1)) break;
//
// So far so good. Is this the last confirmed valid page?
//
if (!--Length)
{
//
// Sanity check that we didn't go out of bounds
//
ASSERT(i != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Loop until all PFN entries have been processed
//
EndPfn = Pfn1 - SizeInPages + 1;
do
{
//
// This PFN is now a used page, set it up
//
MI_SET_USAGE(MI_USAGE_CONTINOUS_ALLOCATION);
MI_SET_PROCESS2("Kernel Driver");
MiUnlinkFreeOrZeroedPage(Pfn1);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u3.e1.StartOfAllocation = 0;
Pfn1->u3.e1.EndOfAllocation = 0;
Pfn1->u3.e1.PrototypePte = 0;
Pfn1->u4.VerifierAllocation = 0;
Pfn1->PteAddress = (PVOID)0xBAADF00D;
//
// Check if this is the last PFN, otherwise go on
//
if (Pfn1 == EndPfn) break;
Pfn1--;
} while (TRUE);
//
// Mark the first and last PFN so we can find them later
//
Pfn1->u3.e1.StartOfAllocation = 1;
(Pfn1 + SizeInPages - 1)->u3.e1.EndOfAllocation = 1;
//
// Now it's safe to let go of the PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// Quick sanity check that the last PFN is consistent
//
EndPfn = Pfn1 + SizeInPages;
ASSERT(EndPfn == MI_PFN_ELEMENT(Page + 1));
//
// Compute the first page, and make sure it's consistent
//
Page = Page - SizeInPages + 1;
ASSERT(Pfn1 == MI_PFN_ELEMENT(Page));
ASSERT(Page != 0);
/* Enable APCs and return the page */
KeLeaveGuardedRegion();
return Page;
}
//
// Keep going. The purpose of this loop is to reconfirm that
// after acquiring the PFN lock these pages are still usable
//
Pfn1++;
Page++;
} while (TRUE);
//
// If we got here, something changed while we hadn't acquired
// the PFN lock yet, so we'll have to restart
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
Length = 0;
}
}
} while (++i != MmPhysicalMemoryBlock->NumberOfRuns);
//
// And if we get here, it means no suitable physical memory runs were found
//
return 0;
}
PFN_COUNT
MiRemovePhysicalPages (
IN PFN_NUMBER StartPage,
IN PFN_NUMBER EndPage
)
/*++
Routine Description:
This routine searches the PFN database for free, zeroed or standby pages
that are marked for removal.
Arguments:
StartPage - Supplies the low physical frame number to remove.
EndPage - Supplies the last physical frame number to remove.
Return Value:
Returns the number of pages removed from the free, zeroed and standby lists.
Environment:
Kernel mode, PFN lock held.
--*/
{
PMMPFN Pfn1;
PMMPFN Pfn2;
PMMPFN PfnNextColored;
PMMPFN PfnNextFlink;
PMMPFN PfnLastColored;
PFN_NUMBER Page;
LOGICAL RemovePage;
ULONG Color;
PMMCOLOR_TABLES ColorHead;
PFN_NUMBER MovedPage;
MMLISTS MemoryList;
PFN_NUMBER PageNextColored;
PFN_NUMBER PageNextFlink;
PFN_NUMBER PageLastColored;
PFN_COUNT NumberOfPages;
PMMPFNLIST ListHead;
LOGICAL RescanNeeded;
MM_PFN_LOCK_ASSERT();
NumberOfPages = 0;
rescan:
//
// Grab all zeroed (and then free) pages first directly from the
// colored lists to avoid multiple walks down these singly linked lists.
// Handle transition pages last.
//
for (MemoryList = ZeroedPageList; MemoryList <= FreePageList; MemoryList += 1) {
ListHead = MmPageLocationList[MemoryList];
for (Color = 0; Color < MmSecondaryColors; Color += 1) {
ColorHead = &MmFreePagesByColor[MemoryList][Color];
MovedPage = MM_EMPTY_LIST;
while (ColorHead->Flink != MM_EMPTY_LIST) {
Page = ColorHead->Flink;
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT ((MMLISTS)Pfn1->u3.e1.PageLocation == MemoryList);
//
// The Flink and Blink must be nonzero here for the page
// to be on the listhead. Only code that scans the
// MmPhysicalMemoryBlock has to check for the zero case.
//
ASSERT (Pfn1->u1.Flink != 0);
ASSERT (Pfn1->u2.Blink != 0);
//
// See if the page is desired by the caller.
//
if (Pfn1->u3.e1.RemovalRequested == 1) {
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
MiUnlinkFreeOrZeroedPage (Page);
MiInsertPageInList (MmPageLocationList[BadPageList],
Page);
NumberOfPages += 1;
}
else {
//
// Unwanted so put the page on the end of list.
// If first time, save pfn.
//
if (MovedPage == MM_EMPTY_LIST) {
MovedPage = Page;
}
else if (Page == MovedPage) {
//
// No more pages available in this colored chain.
//
break;
}
//
// If the colored chain has more than one entry then
// put this page on the end.
//
PageNextColored = (PFN_NUMBER)Pfn1->OriginalPte.u.Long;
if (PageNextColored == MM_EMPTY_LIST) {
//
// No more pages available in this colored chain.
//
break;
}
ASSERT (Pfn1->u1.Flink != 0);
ASSERT (Pfn1->u1.Flink != MM_EMPTY_LIST);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
PfnNextColored = MI_PFN_ELEMENT(PageNextColored);
ASSERT ((MMLISTS)PfnNextColored->u3.e1.PageLocation == MemoryList);
ASSERT (PfnNextColored->PteFrame != MI_MAGIC_AWE_PTEFRAME);
//
// Adjust the free page list so Page
// follows PageNextFlink.
//
PageNextFlink = Pfn1->u1.Flink;
PfnNextFlink = MI_PFN_ELEMENT(PageNextFlink);
ASSERT ((MMLISTS)PfnNextFlink->u3.e1.PageLocation == MemoryList);
ASSERT (PfnNextFlink->PteFrame != MI_MAGIC_AWE_PTEFRAME);
PfnLastColored = ColorHead->Blink;
ASSERT (PfnLastColored != (PMMPFN)MM_EMPTY_LIST);
ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
ASSERT (PfnLastColored->PteFrame != MI_MAGIC_AWE_PTEFRAME);
ASSERT (PfnLastColored->u2.Blink != MM_EMPTY_LIST);
ASSERT ((MMLISTS)PfnLastColored->u3.e1.PageLocation == MemoryList);
PageLastColored = PfnLastColored - MmPfnDatabase;
if (ListHead->Flink == Page) {
ASSERT (Pfn1->u2.Blink == MM_EMPTY_LIST);
ASSERT (ListHead->Blink != Page);
ListHead->Flink = PageNextFlink;
PfnNextFlink->u2.Blink = MM_EMPTY_LIST;
}
else {
ASSERT (Pfn1->u2.Blink != MM_EMPTY_LIST);
ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u3.e1.PageLocation == MemoryList);
MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink = PageNextFlink;
PfnNextFlink->u2.Blink = Pfn1->u2.Blink;
}
#if DBG
if (PfnLastColored->u1.Flink == MM_EMPTY_LIST) {
ASSERT (ListHead->Blink == PageLastColored);
}
#endif
Pfn1->u1.Flink = PfnLastColored->u1.Flink;
Pfn1->u2.Blink = PageLastColored;
if (ListHead->Blink == PageLastColored) {
ListHead->Blink = Page;
}
//
// Adjust the colored chains.
//
if (PfnLastColored->u1.Flink != MM_EMPTY_LIST) {
ASSERT (MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->PteFrame != MI_MAGIC_AWE_PTEFRAME);
ASSERT ((MMLISTS)(MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u3.e1.PageLocation) == MemoryList);
MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u2.Blink = Page;
}
PfnLastColored->u1.Flink = Page;
ColorHead->Flink = PageNextColored;
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
PfnLastColored->OriginalPte.u.Long = Page;
ColorHead->Blink = Pfn1;
}
}
}
}
RescanNeeded = FALSE;
Pfn1 = MI_PFN_ELEMENT (StartPage);
do {
if ((Pfn1->u3.e1.PageLocation == StandbyPageList) &&
(Pfn1->u1.Flink != 0) &&
(Pfn1->u2.Blink != 0) &&
(Pfn1->u3.e2.ReferenceCount == 0)) {
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
RemovePage = TRUE;
if (Pfn1->u3.e1.RemovalRequested == 0) {
//
// This page is not directly needed for a hot remove - but if
// it contains a chunk of prototype PTEs (and this chunk is
// in a page that needs to be removed), then any pages
// referenced by transition prototype PTEs must also be removed
// before the desired page can be removed.
//
// The same analogy holds for page parent, directory and table
// pages.
//
Pfn2 = MI_PFN_ELEMENT (Pfn1->PteFrame);
if (Pfn2->u3.e1.RemovalRequested == 0) {
#if defined (_WIN64)
Pfn2 = MI_PFN_ELEMENT (Pfn2->PteFrame);
if (Pfn2->u3.e1.RemovalRequested == 0) {
RemovePage = FALSE;
}
else if (Pfn2->u2.ShareCount == 1) {
RescanNeeded = TRUE;
}
#else
RemovePage = FALSE;
#endif
}
else if (Pfn2->u2.ShareCount == 1) {
RescanNeeded = TRUE;
}
}
if (RemovePage == TRUE) {
//
// This page is in the desired range - grab it.
//
MiUnlinkPageFromList (Pfn1);
MiRestoreTransitionPte (StartPage);
MiInsertPageInList (MmPageLocationList[BadPageList],
StartPage);
NumberOfPages += 1;
}
}
StartPage += 1;
Pfn1 += 1;
} while (StartPage < EndPage);
if (RescanNeeded == TRUE) {
//
// A page table, directory or parent was freed by removing a transition
// page from the cache. Rescan from the top to pick it up.
//
#if DBG
MiDynmemData[7] += 1;
#endif
goto rescan;
}
#if DBG
else {
MiDynmemData[8] += 1;
}
#endif
return NumberOfPages;
}
