PFN_NUMBER
NTAPI
MiFindContiguousPages(IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN PFN_NUMBER SizeInPages,
IN MEMORY_CACHING_TYPE CacheType)
{
PFN_NUMBER Page, PageCount, LastPage, Length, BoundaryMask;
ULONG i = 0;
PMMPFN Pfn1, EndPfn;
KIRQL OldIrql;
PAGED_CODE();
ASSERT(SizeInPages != 0);
//
// Convert the boundary PFN into an alignment mask
//
BoundaryMask = ~(BoundaryPfn - 1);
/* Disable APCs */
KeEnterGuardedRegion();
//
// Loop all the physical memory blocks
//
do
{
//
// Capture the base page and length of this memory block
//
Page = MmPhysicalMemoryBlock->Run[i].BasePage;
PageCount = MmPhysicalMemoryBlock->Run[i].PageCount;
//
// Check how far this memory block will go
//
LastPage = Page + PageCount;
//
// Trim it down to only the PFNs we're actually interested in
//
if ((LastPage - 1) > HighestPfn) LastPage = HighestPfn + 1;
if (Page < LowestPfn) Page = LowestPfn;
//
// Skip this run if it's empty or fails to contain all the pages we need
//
if (!(PageCount) || ((Page + SizeInPages) > LastPage)) continue;
//
// Now scan all the relevant PFNs in this run
//
Length = 0;
for (Pfn1 = MI_PFN_ELEMENT(Page); Page < LastPage; Page++, Pfn1++)
{
//
// If this PFN is in use, ignore it
//
if (MiIsPfnInUse(Pfn1))
{
Length = 0;
continue;
}
//
// If we haven't chosen a start PFN yet and the caller specified an
// alignment, make sure the page matches the alignment restriction
//
if ((!(Length) && (BoundaryPfn)) &&
(((Page ^ (Page + SizeInPages - 1)) & BoundaryMask)))
{
//
// It does not, so bail out
//
continue;
}
//
// Increase the number of valid pages, and check if we have enough
//
if (++Length == SizeInPages)
{
//
// It appears we've amassed enough legitimate pages, rollback
//
Pfn1 -= (Length - 1);
Page -= (Length - 1);
//
// Acquire the PFN lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// Things might've changed for us. Is the page still free?
//
if (MiIsPfnInUse(Pfn1)) break;
//
// So far so good. Is this the last confirmed valid page?
//
if (!--Length)
{
//
// Sanity check that we didn't go out of bounds
//
ASSERT(i != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Loop until all PFN entries have been processed
//
EndPfn = Pfn1 - SizeInPages + 1;
do
{
//
// This PFN is now a used page, set it up
//
MI_SET_USAGE(MI_USAGE_CONTINOUS_ALLOCATION);
MI_SET_PROCESS2("Kernel Driver");
MiUnlinkFreeOrZeroedPage(Pfn1);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u3.e1.StartOfAllocation = 0;
Pfn1->u3.e1.EndOfAllocation = 0;
Pfn1->u3.e1.PrototypePte = 0;
Pfn1->u4.VerifierAllocation = 0;
Pfn1->PteAddress = (PVOID)0xBAADF00D;
//
// Check if this is the last PFN, otherwise go on
//
if (Pfn1 == EndPfn) break;
Pfn1--;
} while (TRUE);
//
// Mark the first and last PFN so we can find them later
//
Pfn1->u3.e1.StartOfAllocation = 1;
(Pfn1 + SizeInPages - 1)->u3.e1.EndOfAllocation = 1;
//
// Now it's safe to let go of the PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// Quick sanity check that the last PFN is consistent
//
EndPfn = Pfn1 + SizeInPages;
ASSERT(EndPfn == MI_PFN_ELEMENT(Page + 1));
//
// Compute the first page, and make sure it's consistent
//
Page = Page - SizeInPages + 1;
ASSERT(Pfn1 == MI_PFN_ELEMENT(Page));
ASSERT(Page != 0);
/* Enable APCs and return the page */
KeLeaveGuardedRegion();
return Page;
}
//
// Keep going. The purpose of this loop is to reconfirm that
// after acquiring the PFN lock these pages are still usable
//
Pfn1++;
Page++;
} while (TRUE);
//
// If we got here, something changed while we hadn't acquired
// the PFN lock yet, so we'll have to restart
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
Length = 0;
}
}
} while (++i != MmPhysicalMemoryBlock->NumberOfRuns);
//
// And if we get here, it means no suitable physical memory runs were found
//
return 0;
}
PVOID
NTAPI
MiAllocatePoolPages(IN POOL_TYPE PoolType,
IN SIZE_T SizeInBytes)
{
PFN_NUMBER PageFrameNumber;
PFN_COUNT SizeInPages, PageTableCount;
ULONG i;
KIRQL OldIrql;
PLIST_ENTRY NextEntry, NextHead, LastHead;
PMMPTE PointerPte, StartPte;
PMMPDE PointerPde;
ULONG EndAllocation;
MMPTE TempPte;
MMPDE TempPde;
PMMPFN Pfn1;
PVOID BaseVa, BaseVaStart;
PMMFREE_POOL_ENTRY FreeEntry;
PKSPIN_LOCK_QUEUE LockQueue;
//
// Figure out how big the allocation is in pages
//
SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes);
//
// Check for overflow
//
if (SizeInPages == 0)
{
//
// Fail
//
return NULL;
}
//
// Handle paged pool
//
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
{
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Lock the paged pool mutex
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Find some empty allocation space
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
MmPagedPoolInfo.PagedPoolHint);
if (i == 0xFFFFFFFF)
{
//
// Get the page bit count
//
i = ((SizeInPages - 1) / PTE_COUNT) + 1;
DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages);
//
// Check if there is enougn paged pool expansion space left
//
if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
//
// Check if we'll have to expand past the last PTE we have available
//
if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// We can only support this much then
//
PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool);
PageTableCount = (PFN_COUNT)(PointerPde + 1 -
MmPagedPoolInfo.NextPdeForPagedPoolExpansion);
ASSERT(PageTableCount < i);
i = PageTableCount;
}
else
{
//
// Otherwise, there is plenty of space left for this expansion
//
PageTableCount = i;
}
//
// Get the template PDE we'll use to expand
//
TempPde = ValidKernelPde;
//
// Get the first PTE in expansion space
//
PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
BaseVa = MiPdeToPte(PointerPde);
BaseVaStart = BaseVa;
//
// Lock the PFN database and loop pages
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// It should not already be valid
//
ASSERT(PointerPde->u.Hard.Valid == 0);
/* Request a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
/* On PAE/x64 systems, there's no double-buffering */
ASSERT(FALSE);
#else
//
// Save it into our double-buffered system page directory
//
MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;
/* Initialize the PFN */
MiInitializePfnForOtherProcess(PageFrameNumber,
(PMMPTE)PointerPde,
MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]);
/* Write the actual PDE now */
// MI_WRITE_VALID_PDE(PointerPde, TempPde);
#endif
//
// Move on to the next expansion address
//
PointerPde++;
BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
i--;
} while (i > 0);
//
// Release the PFN database lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// These pages are now available, clear their availablity bits
//
EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
PTE_COUNT;
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
EndAllocation,
PageTableCount * PTE_COUNT);
//
// Update the next expansion location
//
MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount;
//
// Zero out the newly available memory
//
RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE);
//
// Now try consuming the pages again
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
0);
if (i == 0xFFFFFFFF)
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
}
//
// Update the pool hint if the request was just one page
//
if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;
//
// Update the end bitmap so we know the bounds of this allocation when
// the time comes to free it
//
EndAllocation = i + SizeInPages - 1;
RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);
//
// Now we can release the lock (it mainly protects the bitmap)
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// Now figure out where this allocation starts
//
BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));
//
// Flush the TLB
//
KeFlushEntireTb(TRUE, TRUE);
/* Setup a demand-zero writable PTE */
MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
//
// Find the first and last PTE, then loop them all
//
PointerPte = MiAddressToPte(BaseVa);
StartPte = PointerPte + SizeInPages;
do
{
//
// Write the demand zero PTE and keep going
//
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
} while (++PointerPte < StartPte);
//
// Return the allocation address to the caller
//
return BaseVa;
}
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Allocations of less than 4 pages go into their individual buckets
//
i = SizeInPages - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
//
// Loop through all the free page lists based on the page index
//
NextHead = &MmNonPagedPoolFreeListHead[i];
LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
//
// Acquire the nonpaged pool lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
do
{
//
// Now loop through all the free page entries in this given list
//
NextEntry = NextHead->Flink;
while (NextEntry != NextHead)
{
/* Is freed non paged pool enabled */
if (MmProtectFreedNonPagedPool)
{
/* We need to be able to touch this page, unprotect it */
MiUnProtectFreeNonPagedPool(NextEntry, 0);
}
//
// Grab the entry and see if it can handle our allocation
//
FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
if (FreeEntry->Size >= SizeInPages)
{
//
// It does, so consume the pages from here
//
FreeEntry->Size -= SizeInPages;
//
// The allocation will begin in this free page area
//
BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
(FreeEntry->Size << PAGE_SHIFT));
/* Remove the item from the list, depending if pool is protected */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolRemoveEntryList(&FreeEntry->List);
else
RemoveEntryList(&FreeEntry->List);
//
// However, check if its' still got space left
//
if (FreeEntry->Size != 0)
{
/* Check which list to insert this entry into */
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
/* Insert the entry into the free list head, check for prot. pool */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
else
InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
//
// Grab the PTE for this allocation
//
PointerPte = MiAddressToPte(BaseVa);
ASSERT(PointerPte->u.Hard.Valid == 1);
//
// Grab the PFN NextEntry and index
//
Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
//
// Now mark it as the beginning of an allocation
//
ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as special pool if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK)
{
Pfn1->u4.VerifierAllocation = 1;
}
//
// Check if the allocation is larger than one page
//
if (SizeInPages != 1)
{
//
// Navigate to the last PFN entry and PTE
//
PointerPte += SizeInPages - 1;
ASSERT(PointerPte->u.Hard.Valid == 1);
Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
}
//
// Mark this PFN as the last (might be the same as the first)
//
ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Release the nonpaged pool lock, and return the allocation
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
return BaseVa;
}
//
// Try the next free page entry
//
NextEntry = FreeEntry->List.Flink;
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
} while (++NextHead < LastHead);
//
// If we got here, we're out of space.
// Start by releasing the lock
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Allocate some system PTEs
//
StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
PointerPte = StartPte;
if (StartPte == NULL)
{
//
// Ran out of memory
//
DPRINT1("Out of NP Expansion Pool\n");
return NULL;
}
//
// Acquire the pool lock now
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
//
// Lock the PFN database too
//
LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
//
// Loop the pages
//
TempPte = ValidKernelPte;
do
{
/* Allocate a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->PteAddress = PointerPte;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u4.VerifierAllocation = 0;
/* Write the PTE for it */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
MI_WRITE_VALID_PTE(PointerPte++, TempPte);
} while (--SizeInPages > 0);
//
// This is the last page
//
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Get the first page and mark it as such
//
Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as a verifier allocation if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1;
//
// Release the PFN and nonpaged pool lock
//
KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Return the address
//
return MiPteToAddress(StartPte);
}
VOID
NTAPI
MmZeroPageThread(VOID)
{
PKTHREAD Thread = KeGetCurrentThread();
PVOID StartAddress, EndAddress;
PVOID WaitObjects[2];
KIRQL OldIrql;
PVOID ZeroAddress;
PFN_NUMBER PageIndex, FreePage;
PMMPFN Pfn1;
/* Get the discardable sections to free them */
MiFindInitializationCode(&StartAddress, &EndAddress);
if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress);
DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed);
/* Set our priority to 0 */
Thread->BasePriority = 0;
KeSetPriorityThread(Thread, 0);
/* Setup the wait objects */
WaitObjects[0] = &MmZeroingPageEvent;
// WaitObjects[1] = &PoSystemIdleTimer; FIXME: Implement idle timer
while (TRUE)
{
KeWaitForMultipleObjects(1, // 2
WaitObjects,
WaitAny,
WrFreePage,
KernelMode,
FALSE,
NULL,
NULL);
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
while (TRUE)
{
if (!MmFreePageListHead.Total)
{
MmZeroingPageThreadActive = FALSE;
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
break;
}
PageIndex = MmFreePageListHead.Flink;
ASSERT(PageIndex != LIST_HEAD);
Pfn1 = MiGetPfnEntry(PageIndex);
MI_SET_USAGE(MI_USAGE_ZERO_LOOP);
MI_SET_PROCESS2("Kernel 0 Loop");
FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex));
/* The first global free page should also be the first on its own list */
if (FreePage != PageIndex)
{
KeBugCheckEx(PFN_LIST_CORRUPT,
0x8F,
FreePage,
PageIndex,
0);
}
Pfn1->u1.Flink = LIST_HEAD;
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
ZeroAddress = MiMapPagesInZeroSpace(Pfn1, 1);
ASSERT(ZeroAddress);
RtlZeroMemory(ZeroAddress, PAGE_SIZE);
MiUnmapPagesInZeroSpace(ZeroAddress, 1);
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
MiInsertPageInList(&MmZeroedPageListHead, PageIndex);
}
}
}
