PVOID
NTAPI
MiFindContiguousMemory(IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN PFN_NUMBER SizeInPages,
IN MEMORY_CACHING_TYPE CacheType)
{
PFN_NUMBER Page;
PHYSICAL_ADDRESS PhysicalAddress;
PMMPFN Pfn1, EndPfn;
PMMPTE PointerPte;
PVOID BaseAddress;
PAGED_CODE();
ASSERT(SizeInPages != 0);
//
// Our last hope is to scan the free page list for contiguous pages
//
Page = MiFindContiguousPages(LowestPfn,
HighestPfn,
BoundaryPfn,
SizeInPages,
CacheType);
if (!Page) return NULL;
//
// We'll just piggyback on the I/O memory mapper
//
PhysicalAddress.QuadPart = Page << PAGE_SHIFT;
BaseAddress = MmMapIoSpace(PhysicalAddress, SizeInPages << PAGE_SHIFT, CacheType);
ASSERT(BaseAddress);
/* Loop the PFN entries */
Pfn1 = MiGetPfnEntry(Page);
EndPfn = Pfn1 + SizeInPages;
PointerPte = MiAddressToPte(BaseAddress);
do
{
/* Write the PTE address */
Pfn1->PteAddress = PointerPte;
Pfn1->u4.PteFrame = PFN_FROM_PTE(MiAddressToPte(PointerPte++));
} while (++Pfn1 < EndPfn);
/* Return the address */
return BaseAddress;
}
/*
* @implemented
*/
VOID
NTAPI
MmUnmapIoSpace(IN PVOID BaseAddress,
IN SIZE_T NumberOfBytes)
{
PFN_NUMBER Pfn;
PFN_COUNT PageCount;
PMMPTE PointerPte;
//
// Sanity check
//
ASSERT(NumberOfBytes != 0);
//
// Get the page count
//
PageCount = ADDRESS_AND_SIZE_TO_SPAN_PAGES(BaseAddress, NumberOfBytes);
//
// Get the PTE and PFN
//
PointerPte = MiAddressToPte(BaseAddress);
Pfn = PFN_FROM_PTE(PointerPte);
//
// Is this an I/O mapping?
//
if (!MiGetPfnEntry(Pfn))
{
//
// Destroy the PTE
//
RtlZeroMemory(PointerPte, PageCount * sizeof(MMPTE));
//
// Blow the TLB
//
KeFlushEntireTb(TRUE, TRUE);
}
//
// Release the PTEs
//
MiReleaseSystemPtes(PointerPte, PageCount, 0);
}
PVOID
NTAPI
MiAllocatePoolPages(IN POOL_TYPE PoolType,
IN SIZE_T SizeInBytes)
{
PFN_NUMBER PageFrameNumber;
PFN_COUNT SizeInPages, PageTableCount;
ULONG i;
KIRQL OldIrql;
PLIST_ENTRY NextEntry, NextHead, LastHead;
PMMPTE PointerPte, StartPte;
PMMPDE PointerPde;
ULONG EndAllocation;
MMPTE TempPte;
MMPDE TempPde;
PMMPFN Pfn1;
PVOID BaseVa, BaseVaStart;
PMMFREE_POOL_ENTRY FreeEntry;
PKSPIN_LOCK_QUEUE LockQueue;
//
// Figure out how big the allocation is in pages
//
SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes);
//
// Check for overflow
//
if (SizeInPages == 0)
{
//
// Fail
//
return NULL;
}
//
// Handle paged pool
//
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
{
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Lock the paged pool mutex
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Find some empty allocation space
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
MmPagedPoolInfo.PagedPoolHint);
if (i == 0xFFFFFFFF)
{
//
// Get the page bit count
//
i = ((SizeInPages - 1) / PTE_COUNT) + 1;
DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages);
//
// Check if there is enougn paged pool expansion space left
//
if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
//
// Check if we'll have to expand past the last PTE we have available
//
if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// We can only support this much then
//
PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool);
PageTableCount = (PFN_COUNT)(PointerPde + 1 -
MmPagedPoolInfo.NextPdeForPagedPoolExpansion);
ASSERT(PageTableCount < i);
i = PageTableCount;
}
else
{
//
// Otherwise, there is plenty of space left for this expansion
//
PageTableCount = i;
}
//
// Get the template PDE we'll use to expand
//
TempPde = ValidKernelPde;
//
// Get the first PTE in expansion space
//
PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
BaseVa = MiPdeToPte(PointerPde);
BaseVaStart = BaseVa;
//
// Lock the PFN database and loop pages
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// It should not already be valid
//
ASSERT(PointerPde->u.Hard.Valid == 0);
/* Request a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
/* On PAE/x64 systems, there's no double-buffering */
ASSERT(FALSE);
#else
//
// Save it into our double-buffered system page directory
//
MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;
/* Initialize the PFN */
MiInitializePfnForOtherProcess(PageFrameNumber,
(PMMPTE)PointerPde,
MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]);
/* Write the actual PDE now */
// MI_WRITE_VALID_PDE(PointerPde, TempPde);
#endif
//
// Move on to the next expansion address
//
PointerPde++;
BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
i--;
} while (i > 0);
//
// Release the PFN database lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// These pages are now available, clear their availablity bits
//
EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
PTE_COUNT;
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
EndAllocation,
PageTableCount * PTE_COUNT);
//
// Update the next expansion location
//
MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount;
//
// Zero out the newly available memory
//
RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE);
//
// Now try consuming the pages again
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
0);
if (i == 0xFFFFFFFF)
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
}
//
// Update the pool hint if the request was just one page
//
if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;
//
// Update the end bitmap so we know the bounds of this allocation when
// the time comes to free it
//
EndAllocation = i + SizeInPages - 1;
RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);
//
// Now we can release the lock (it mainly protects the bitmap)
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// Now figure out where this allocation starts
//
BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));
//
// Flush the TLB
//
KeFlushEntireTb(TRUE, TRUE);
/* Setup a demand-zero writable PTE */
MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
//
// Find the first and last PTE, then loop them all
//
PointerPte = MiAddressToPte(BaseVa);
StartPte = PointerPte + SizeInPages;
do
{
//
// Write the demand zero PTE and keep going
//
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
} while (++PointerPte < StartPte);
//
// Return the allocation address to the caller
//
return BaseVa;
}
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Allocations of less than 4 pages go into their individual buckets
//
i = SizeInPages - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
//
// Loop through all the free page lists based on the page index
//
NextHead = &MmNonPagedPoolFreeListHead[i];
LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
//
// Acquire the nonpaged pool lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
do
{
//
// Now loop through all the free page entries in this given list
//
NextEntry = NextHead->Flink;
while (NextEntry != NextHead)
{
/* Is freed non paged pool enabled */
if (MmProtectFreedNonPagedPool)
{
/* We need to be able to touch this page, unprotect it */
MiUnProtectFreeNonPagedPool(NextEntry, 0);
}
//
// Grab the entry and see if it can handle our allocation
//
FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
if (FreeEntry->Size >= SizeInPages)
{
//
// It does, so consume the pages from here
//
FreeEntry->Size -= SizeInPages;
//
// The allocation will begin in this free page area
//
BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
(FreeEntry->Size << PAGE_SHIFT));
/* Remove the item from the list, depending if pool is protected */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolRemoveEntryList(&FreeEntry->List);
else
RemoveEntryList(&FreeEntry->List);
//
// However, check if its' still got space left
//
if (FreeEntry->Size != 0)
{
/* Check which list to insert this entry into */
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
/* Insert the entry into the free list head, check for prot. pool */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
else
InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
//
// Grab the PTE for this allocation
//
PointerPte = MiAddressToPte(BaseVa);
ASSERT(PointerPte->u.Hard.Valid == 1);
//
// Grab the PFN NextEntry and index
//
Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
//
// Now mark it as the beginning of an allocation
//
ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as special pool if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK)
{
Pfn1->u4.VerifierAllocation = 1;
}
//
// Check if the allocation is larger than one page
//
if (SizeInPages != 1)
{
//
// Navigate to the last PFN entry and PTE
//
PointerPte += SizeInPages - 1;
ASSERT(PointerPte->u.Hard.Valid == 1);
Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
}
//
// Mark this PFN as the last (might be the same as the first)
//
ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Release the nonpaged pool lock, and return the allocation
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
return BaseVa;
}
//
// Try the next free page entry
//
NextEntry = FreeEntry->List.Flink;
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
} while (++NextHead < LastHead);
//
// If we got here, we're out of space.
// Start by releasing the lock
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Allocate some system PTEs
//
StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
PointerPte = StartPte;
if (StartPte == NULL)
{
//
// Ran out of memory
//
DPRINT1("Out of NP Expansion Pool\n");
return NULL;
}
//
// Acquire the pool lock now
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
//
// Lock the PFN database too
//
LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
//
// Loop the pages
//
TempPte = ValidKernelPte;
do
{
/* Allocate a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->PteAddress = PointerPte;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u4.VerifierAllocation = 0;
/* Write the PTE for it */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
MI_WRITE_VALID_PTE(PointerPte++, TempPte);
} while (--SizeInPages > 0);
//
// This is the last page
//
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Get the first page and mark it as such
//
Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as a verifier allocation if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1;
//
// Release the PFN and nonpaged pool lock
//
KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Return the address
//
return MiPteToAddress(StartPte);
}
VOID
NTAPI
INIT_FUNCTION
MiInitializeNonPagedPool(VOID)
{
ULONG i;
PFN_COUNT PoolPages;
PMMFREE_POOL_ENTRY FreeEntry, FirstEntry;
PMMPTE PointerPte;
PAGED_CODE();
//
// Initialize the pool S-LISTs as well as their maximum count. In general,
// we'll allow 8 times the default on a 2GB system, and two times the default
// on a 1GB system.
//
InitializeSListHead(&MiPagedPoolSListHead);
InitializeSListHead(&MiNonPagedPoolSListHead);
if (MmNumberOfPhysicalPages >= ((2 * _1GB) /PAGE_SIZE))
{
MiNonPagedPoolSListMaximum *= 8;
MiPagedPoolSListMaximum *= 8;
}
else if (MmNumberOfPhysicalPages >= (_1GB /PAGE_SIZE))
{
MiNonPagedPoolSListMaximum *= 2;
MiPagedPoolSListMaximum *= 2;
}
//
// However if debugging options for the pool are enabled, turn off the S-LIST
// to reduce the risk of messing things up even more
//
if (MmProtectFreedNonPagedPool)
{
MiNonPagedPoolSListMaximum = 0;
MiPagedPoolSListMaximum = 0;
}
//
// We keep 4 lists of free pages (4 lists help avoid contention)
//
for (i = 0; i < MI_MAX_FREE_PAGE_LISTS; i++)
{
//
// Initialize each of them
//
InitializeListHead(&MmNonPagedPoolFreeListHead[i]);
}
//
// Calculate how many pages the initial nonpaged pool has
//
PoolPages = (PFN_COUNT)BYTES_TO_PAGES(MmSizeOfNonPagedPoolInBytes);
MmNumberOfFreeNonPagedPool = PoolPages;
//
// Initialize the first free entry
//
FreeEntry = MmNonPagedPoolStart;
FirstEntry = FreeEntry;
FreeEntry->Size = PoolPages;
FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
FreeEntry->Owner = FirstEntry;
//
// Insert it into the last list
//
InsertHeadList(&MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS - 1],
&FreeEntry->List);
//
// Now create free entries for every single other page
//
while (PoolPages-- > 1)
{
//
// Link them all back to the original entry
//
FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)FreeEntry + PAGE_SIZE);
FreeEntry->Owner = FirstEntry;
FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
}
//
// Validate and remember first allocated pool page
//
PointerPte = MiAddressToPte(MmNonPagedPoolStart);
ASSERT(PointerPte->u.Hard.Valid == 1);
MiStartOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);
//
// Keep track of where initial nonpaged pool ends
//
MmNonPagedPoolEnd0 = (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
MmSizeOfNonPagedPoolInBytes);
//
// Validate and remember last allocated pool page
//
PointerPte = MiAddressToPte((PVOID)((ULONG_PTR)MmNonPagedPoolEnd0 - 1));
ASSERT(PointerPte->u.Hard.Valid == 1);
MiEndOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);
//
// Validate the first nonpaged pool expansion page (which is a guard page)
//
PointerPte = MiAddressToPte(MmNonPagedPoolExpansionStart);
ASSERT(PointerPte->u.Hard.Valid == 0);
//
// Calculate the size of the expansion region alone
//
MiExpansionPoolPagesInitialCharge = (PFN_COUNT)
BYTES_TO_PAGES(MmMaximumNonPagedPoolInBytes - MmSizeOfNonPagedPoolInBytes);
//
// Remove 2 pages, since there's a guard page on top and on the bottom
//
MiExpansionPoolPagesInitialCharge -= 2;
//
// Now initialize the nonpaged pool expansion PTE space. Remember there's a
// guard page on top so make sure to skip it. The bottom guard page will be
// guaranteed by the fact our size is off by one.
//
MiInitializeSystemPtes(PointerPte + 1,
MiExpansionPoolPagesInitialCharge,
NonPagedPoolExpansion);
}
/*
* @implemented
*/
VOID
NTAPI
MmBuildMdlForNonPagedPool(IN PMDL Mdl)
{
PPFN_NUMBER MdlPages, EndPage;
PFN_NUMBER Pfn, PageCount;
PVOID Base;
PMMPTE PointerPte;
//
// Sanity checks
//
ASSERT(Mdl->ByteCount != 0);
ASSERT((Mdl->MdlFlags & (MDL_PAGES_LOCKED |
MDL_MAPPED_TO_SYSTEM_VA |
MDL_SOURCE_IS_NONPAGED_POOL |
MDL_PARTIAL)) == 0);
//
// We know the MDL isn't associated to a process now
//
Mdl->Process = NULL;
//
// Get page and VA information
//
MdlPages = (PPFN_NUMBER)(Mdl + 1);
Base = Mdl->StartVa;
//
// Set the system address and now get the page count
//
Mdl->MappedSystemVa = (PVOID)((ULONG_PTR)Base + Mdl->ByteOffset);
PageCount = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Mdl->MappedSystemVa,
Mdl->ByteCount);
ASSERT(PageCount != 0);
EndPage = MdlPages + PageCount;
//
// Loop the PTEs
//
PointerPte = MiAddressToPte(Base);
do
{
//
// Write the PFN
//
Pfn = PFN_FROM_PTE(PointerPte++);
*MdlPages++ = Pfn;
} while (MdlPages < EndPage);
//
// Set the nonpaged pool flag
//
Mdl->MdlFlags |= MDL_SOURCE_IS_NONPAGED_POOL;
//
// Check if this is an I/O mapping
//
if (!MiGetPfnEntry(Pfn)) Mdl->MdlFlags |= MDL_IO_SPACE;
}
/*
* @implemented
*/
VOID
NTAPI
MmProbeAndLockPages(IN PMDL Mdl,
IN KPROCESSOR_MODE AccessMode,
IN LOCK_OPERATION Operation)
{
PPFN_NUMBER MdlPages;
PVOID Base, Address, LastAddress, StartAddress;
ULONG LockPages, TotalPages;
NTSTATUS Status = STATUS_SUCCESS;
PEPROCESS CurrentProcess;
NTSTATUS ProbeStatus;
PMMPTE PointerPte, LastPte;
PMMPDE PointerPde;
#if (_MI_PAGING_LEVELS >= 3)
PMMPDE PointerPpe;
#endif
#if (_MI_PAGING_LEVELS == 4)
PMMPDE PointerPxe;
#endif
PFN_NUMBER PageFrameIndex;
BOOLEAN UsePfnLock;
KIRQL OldIrql;
PMMPFN Pfn1;
DPRINT("Probing MDL: %p\n", Mdl);
//
// Sanity checks
//
ASSERT(Mdl->ByteCount != 0);
ASSERT(((ULONG)Mdl->ByteOffset & ~(PAGE_SIZE - 1)) == 0);
ASSERT(((ULONG_PTR)Mdl->StartVa & (PAGE_SIZE - 1)) == 0);
ASSERT((Mdl->MdlFlags & (MDL_PAGES_LOCKED |
MDL_MAPPED_TO_SYSTEM_VA |
MDL_SOURCE_IS_NONPAGED_POOL |
MDL_PARTIAL |
MDL_IO_SPACE)) == 0);
//
// Get page and base information
//
MdlPages = (PPFN_NUMBER)(Mdl + 1);
Base = Mdl->StartVa;
//
// Get the addresses and how many pages we span (and need to lock)
//
Address = (PVOID)((ULONG_PTR)Base + Mdl->ByteOffset);
LastAddress = (PVOID)((ULONG_PTR)Address + Mdl->ByteCount);
LockPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Address, Mdl->ByteCount);
ASSERT(LockPages != 0);
/* Block invalid access */
if ((AccessMode != KernelMode) &&
((LastAddress > (PVOID)MM_USER_PROBE_ADDRESS) || (Address >= LastAddress)))
{
/* Caller should be in SEH, raise the error */
*MdlPages = LIST_HEAD;
ExRaiseStatus(STATUS_ACCESS_VIOLATION);
}
//
// Get the process
//
if (Address <= MM_HIGHEST_USER_ADDRESS)
{
//
// Get the process
//
CurrentProcess = PsGetCurrentProcess();
}
else
{
//
// No process
//
CurrentProcess = NULL;
}
//
// Save the number of pages we'll have to lock, and the start address
//
TotalPages = LockPages;
StartAddress = Address;
/* Large pages not supported */
ASSERT(!MI_IS_PHYSICAL_ADDRESS(Address));
//
// Now probe them
//
ProbeStatus = STATUS_SUCCESS;
_SEH2_TRY
{
//
// Enter probe loop
//
do
{
//
// Assume failure
//
*MdlPages = LIST_HEAD;
//
// Read
//
*(volatile CHAR*)Address;
//
// Check if this is write access (only probe for user-mode)
//
if ((Operation != IoReadAccess) &&
(Address <= MM_HIGHEST_USER_ADDRESS))
{
//
// Probe for write too
//
ProbeForWriteChar(Address);
}
//
// Next address...
//
Address = PAGE_ALIGN((ULONG_PTR)Address + PAGE_SIZE);
//
// Next page...
//
LockPages--;
MdlPages++;
} while (Address < LastAddress);
//
// Reset back to the original page
//
ASSERT(LockPages == 0);
MdlPages = (PPFN_NUMBER)(Mdl + 1);
}
_SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
{
//
// Oops :(
//
ProbeStatus = _SEH2_GetExceptionCode();
}
_SEH2_END;
//
// So how did that go?
//
if (ProbeStatus != STATUS_SUCCESS)
{
//
// Fail
//
DPRINT1("MDL PROBE FAILED!\n");
Mdl->Process = NULL;
ExRaiseStatus(ProbeStatus);
}
//
// Get the PTE and PDE
//
PointerPte = MiAddressToPte(StartAddress);
PointerPde = MiAddressToPde(StartAddress);
#if (_MI_PAGING_LEVELS >= 3)
PointerPpe = MiAddressToPpe(StartAddress);
#endif
#if (_MI_PAGING_LEVELS == 4)
PointerPxe = MiAddressToPxe(StartAddress);
#endif
//
// Sanity check
//
ASSERT(MdlPages == (PPFN_NUMBER)(Mdl + 1));
//
// Check what kind of operation this is
//
if (Operation != IoReadAccess)
{
//
// Set the write flag
//
Mdl->MdlFlags |= MDL_WRITE_OPERATION;
}
else
{
//
// Remove the write flag
//
Mdl->MdlFlags &= ~(MDL_WRITE_OPERATION);
}
//
// Mark the MDL as locked *now*
//
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
//
// Check if this came from kernel mode
//
if (Base > MM_HIGHEST_USER_ADDRESS)
{
//
// We should not have a process
//
ASSERT(CurrentProcess == NULL);
Mdl->Process = NULL;
//
// In kernel mode, we don't need to check for write access
//
Operation = IoReadAccess;
//
// Use the PFN lock
//
UsePfnLock = TRUE;
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
}
else
{
//
// Sanity checks
//
ASSERT(TotalPages != 0);
ASSERT(CurrentProcess == PsGetCurrentProcess());
//
// Track locked pages
//
InterlockedExchangeAddSizeT(&CurrentProcess->NumberOfLockedPages,
TotalPages);
//
// Save the process
//
Mdl->Process = CurrentProcess;
/* Lock the process working set */
MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
UsePfnLock = FALSE;
OldIrql = MM_NOIRQL;
}
//
// Get the last PTE
//
LastPte = MiAddressToPte((PVOID)((ULONG_PTR)LastAddress - 1));
//
// Loop the pages
//
do
{
//
// Assume failure and check for non-mapped pages
//
*MdlPages = LIST_HEAD;
while (
#if (_MI_PAGING_LEVELS == 4)
(PointerPxe->u.Hard.Valid == 0) ||
#endif
#if (_MI_PAGING_LEVELS >= 3)
(PointerPpe->u.Hard.Valid == 0) ||
#endif
(PointerPde->u.Hard.Valid == 0) ||
(PointerPte->u.Hard.Valid == 0))
{
//
// What kind of lock were we using?
//
if (UsePfnLock)
{
//
// Release PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
else
{
/* Release process working set */
MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
}
//
// Access the page
//
Address = MiPteToAddress(PointerPte);
//HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked
Status = MmAccessFault(FALSE, Address, KernelMode, (PVOID)0xBADBADA3);
if (!NT_SUCCESS(Status))
{
//
// Fail
//
DPRINT1("Access fault failed\n");
goto Cleanup;
}
//
// What lock should we use?
//
if (UsePfnLock)
{
//
// Grab the PFN lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
}
else
{
/* Lock the process working set */
MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
}
}
//
// Check if this was a write or modify
//
if (Operation != IoReadAccess)
{
//
// Check if the PTE is not writable
//
if (MI_IS_PAGE_WRITEABLE(PointerPte) == FALSE)
{
//
// Check if it's copy on write
//
if (MI_IS_PAGE_COPY_ON_WRITE(PointerPte))
{
//
// Get the base address and allow a change for user-mode
//
Address = MiPteToAddress(PointerPte);
if (Address <= MM_HIGHEST_USER_ADDRESS)
{
//
// What kind of lock were we using?
//
if (UsePfnLock)
{
//
// Release PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
else
{
/* Release process working set */
MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
}
//
// Access the page
//
//HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked
Status = MmAccessFault(TRUE, Address, KernelMode, (PVOID)0xBADBADA3);
if (!NT_SUCCESS(Status))
{
//
// Fail
//
DPRINT1("Access fault failed\n");
goto Cleanup;
}
//
// Re-acquire the lock
//
if (UsePfnLock)
{
//
// Grab the PFN lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
}
else
{
/* Lock the process working set */
MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
}
//
// Start over
//
continue;
}
}
//
// Fail, since we won't allow this
//
Status = STATUS_ACCESS_VIOLATION;
goto CleanupWithLock;
}
}
//
// Grab the PFN
//
PageFrameIndex = PFN_FROM_PTE(PointerPte);
Pfn1 = MiGetPfnEntry(PageFrameIndex);
if (Pfn1)
{
/* Either this is for kernel-mode, or the working set is held */
ASSERT((CurrentProcess == NULL) || (UsePfnLock == FALSE));
/* No Physical VADs supported yet */
if (CurrentProcess) ASSERT(CurrentProcess->PhysicalVadRoot == NULL);
/* This address should already exist and be fully valid */
MiReferenceProbedPageAndBumpLockCount(Pfn1);
}
else
{
//
// For I/O addresses, just remember this
//
Mdl->MdlFlags |= MDL_IO_SPACE;
}
//
// Write the page and move on
//
*MdlPages++ = PageFrameIndex;
PointerPte++;
/* Check if we're on a PDE boundary */
if (MiIsPteOnPdeBoundary(PointerPte)) PointerPde++;
#if (_MI_PAGING_LEVELS >= 3)
if (MiIsPteOnPpeBoundary(PointerPte)) PointerPpe++;
#endif
#if (_MI_PAGING_LEVELS == 4)
if (MiIsPteOnPxeBoundary(PointerPte)) PointerPxe++;
#endif
} while (PointerPte <= LastPte);
//
// What kind of lock were we using?
//
if (UsePfnLock)
{
//
// Release PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
else
{
/* Release process working set */
MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
}
//
// Sanity check
//
ASSERT((Mdl->MdlFlags & MDL_DESCRIBES_AWE) == 0);
return;
CleanupWithLock:
//
// This is the failure path
//
ASSERT(!NT_SUCCESS(Status));
//
// What kind of lock were we using?
//
if (UsePfnLock)
{
//
// Release PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
else
{
/* Release process working set */
MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
}
Cleanup:
//
// Pages must be locked so MmUnlock can work
//
ASSERT(Mdl->MdlFlags & MDL_PAGES_LOCKED);
MmUnlockPages(Mdl);
//
// Raise the error
//
ExRaiseStatus(Status);
}
BOOLEAN
NTAPI
INIT_FUNCTION
MmInitSystem(IN ULONG Phase,
IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
extern MMPTE ValidKernelPte;
PMMPTE PointerPte;
MMPTE TempPte = ValidKernelPte;
PFN_NUMBER PageFrameNumber;
/* Initialize the kernel address space */
ASSERT(Phase == 1);
KeInitializeGuardedMutex(&PsIdleProcess->AddressCreationLock);
MmKernelAddressSpace = &PsIdleProcess->Vm;
/* Intialize system memory areas */
MiInitSystemMemoryAreas();
/* Dump the address space */
MiDbgDumpAddressSpace();
MmInitGlobalKernelPageDirectory();
MiInitializeUserPfnBitmap();
MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory);
MmInitializeRmapList();
MmInitializePageOp();
MmInitSectionImplementation();
MmInitPagingFile();
//
// Create a PTE to double-map the shared data section. We allocate it
// from paged pool so that we can't fault when trying to touch the PTE
// itself (to map it), since paged pool addresses will already be mapped
// by the fault handler.
//
MmSharedUserDataPte = ExAllocatePoolWithTag(PagedPool,
sizeof(MMPTE),
' mM');
if (!MmSharedUserDataPte) return FALSE;
//
// Now get the PTE for shared data, and read the PFN that holds it
//
PointerPte = MiAddressToPte((PVOID)KI_USER_SHARED_DATA);
ASSERT(PointerPte->u.Hard.Valid == 1);
PageFrameNumber = PFN_FROM_PTE(PointerPte);
/* Build the PTE and write it */
MI_MAKE_HARDWARE_PTE_KERNEL(&TempPte,
PointerPte,
MM_READONLY,
PageFrameNumber);
*MmSharedUserDataPte = TempPte;
/* Setup the memory threshold events */
if (!MiInitializeMemoryEvents()) return FALSE;
/*
* Unmap low memory
*/
MiInitBalancerThread();
/*
* Initialise the modified page writer.
*/
MmInitMpwThread();
/* Initialize the balance set manager */
MmInitBsmThread();
return TRUE;
}
VOID
NTAPI
MiFreeContiguousMemory(IN PVOID BaseAddress)
{
KIRQL OldIrql;
PFN_NUMBER PageFrameIndex, LastPage, PageCount;
PMMPFN Pfn1, StartPfn;
PMMPTE PointerPte;
PAGED_CODE();
//
// First, check if the memory came from initial nonpaged pool, or expansion
//
if (((BaseAddress >= MmNonPagedPoolStart) &&
(BaseAddress < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
MmSizeOfNonPagedPoolInBytes))) ||
((BaseAddress >= MmNonPagedPoolExpansionStart) &&
(BaseAddress < MmNonPagedPoolEnd)))
{
//
// It did, so just use the pool to free this
//
ExFreePoolWithTag(BaseAddress, 'mCmM');
return;
}
/* Get the PTE and frame number for the allocation*/
PointerPte = MiAddressToPte(BaseAddress);
PageFrameIndex = PFN_FROM_PTE(PointerPte);
//
// Now get the PFN entry for this, and make sure it's the correct one
//
Pfn1 = MiGetPfnEntry(PageFrameIndex);
if ((!Pfn1) || (Pfn1->u3.e1.StartOfAllocation == 0))
{
//
// This probably means you did a free on an address that was in between
//
KeBugCheckEx(BAD_POOL_CALLER,
0x60,
(ULONG_PTR)BaseAddress,
0,
0);
}
//
// Now this PFN isn't the start of any allocation anymore, it's going out
//
StartPfn = Pfn1;
Pfn1->u3.e1.StartOfAllocation = 0;
/* Loop the PFNs until we find the one that marks the end of the allocation */
do
{
/* Make sure these are the pages we setup in the allocation routine */
ASSERT(Pfn1->u3.e2.ReferenceCount == 1);
ASSERT(Pfn1->u2.ShareCount == 1);
ASSERT(Pfn1->PteAddress == PointerPte);
ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
ASSERT(Pfn1->u4.VerifierAllocation == 0);
ASSERT(Pfn1->u3.e1.PrototypePte == 0);
/* Set the special pending delete marker */
MI_SET_PFN_DELETED(Pfn1);
/* Keep going for assertions */
PointerPte++;
} while (Pfn1++->u3.e1.EndOfAllocation == 0);
//
// Found it, unmark it
//
Pfn1--;
Pfn1->u3.e1.EndOfAllocation = 0;
//
// Now compute how many pages this represents
//
PageCount = (ULONG)(Pfn1 - StartPfn + 1);
//
// So we can know how much to unmap (recall we piggyback on I/O mappings)
//
MmUnmapIoSpace(BaseAddress, PageCount << PAGE_SHIFT);
//
// Lock the PFN database
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
//
// Loop all the pages
//
LastPage = PageFrameIndex + PageCount;
Pfn1 = MiGetPfnEntry(PageFrameIndex);
do
{
/* Decrement the share count and move on */
MiDecrementShareCount(Pfn1++, PageFrameIndex++);
} while (PageFrameIndex < LastPage);
//
// Release the PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
PVOID
NTAPI
MiCheckForContiguousMemory(IN PVOID BaseAddress,
IN PFN_NUMBER BaseAddressPages,
IN PFN_NUMBER SizeInPages,
IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute)
{
PMMPTE StartPte, EndPte;
PFN_NUMBER PreviousPage = 0, Page, HighPage, BoundaryMask, Pages = 0;
//
// Okay, first of all check if the PFNs match our restrictions
//
if (LowestPfn > HighestPfn) return NULL;
if (LowestPfn + SizeInPages <= LowestPfn) return NULL;
if (LowestPfn + SizeInPages - 1 > HighestPfn) return NULL;
if (BaseAddressPages < SizeInPages) return NULL;
//
// This is the last page we need to get to and the boundary requested
//
HighPage = HighestPfn + 1 - SizeInPages;
BoundaryMask = ~(BoundaryPfn - 1);
//
// And here's the PTEs for this allocation. Let's go scan them.
//
StartPte = MiAddressToPte(BaseAddress);
EndPte = StartPte + BaseAddressPages;
while (StartPte < EndPte)
{
//
// Get this PTE's page number
//
ASSERT (StartPte->u.Hard.Valid == 1);
Page = PFN_FROM_PTE(StartPte);
//
// Is this the beginning of our adventure?
//
if (!Pages)
{
//
// Check if this PFN is within our range
//
if ((Page >= LowestPfn) && (Page <= HighPage))
{
//
// It is! Do you care about boundary (alignment)?
//
if (!(BoundaryPfn) ||
(!((Page ^ (Page + SizeInPages - 1)) & BoundaryMask)))
{
//
// You don't care, or you do care but we deliver
//
Pages++;
}
}
//
// Have we found all the pages we need by now?
// Incidently, this means you only wanted one page
//
if (Pages == SizeInPages)
{
//
// Mission complete
//
return MiPteToAddress(StartPte);
}
}
else
{
//
// Have we found a page that doesn't seem to be contiguous?
//
if (Page != (PreviousPage + 1))
{
//
// Ah crap, we have to start over
//
Pages = 0;
continue;
}
//
// Otherwise, we're still in the game. Do we have all our pages?
//
if (++Pages == SizeInPages)
{
//
// We do! This entire range was contiguous, so we'll return it!
//
return MiPteToAddress(StartPte - Pages + 1);
}
}
//
// Try with the next PTE, remember this PFN
//
PreviousPage = Page;
StartPte++;
continue;
}
//
// All good returns are within the loop...
//
return NULL;
}
VOID
NTAPI
INIT_FUNCTION
MiInitializeNonPagedPool(VOID)
{
ULONG i;
PFN_NUMBER PoolPages;
PMMFREE_POOL_ENTRY FreeEntry, FirstEntry;
PMMPTE PointerPte;
PAGED_CODE();
//
// We keep 4 lists of free pages (4 lists help avoid contention)
//
for (i = 0; i < MI_MAX_FREE_PAGE_LISTS; i++)
{
//
// Initialize each of them
//
InitializeListHead(&MmNonPagedPoolFreeListHead[i]);
}
//
// Calculate how many pages the initial nonpaged pool has
//
PoolPages = BYTES_TO_PAGES(MmSizeOfNonPagedPoolInBytes);
MmNumberOfFreeNonPagedPool = PoolPages;
//
// Initialize the first free entry
//
FreeEntry = MmNonPagedPoolStart;
FirstEntry = FreeEntry;
FreeEntry->Size = PoolPages;
FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
FreeEntry->Owner = FirstEntry;
//
// Insert it into the last list
//
InsertHeadList(&MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS - 1],
&FreeEntry->List);
//
// Now create free entries for every single other page
//
while (PoolPages-- > 1)
{
//
// Link them all back to the original entry
//
FreeEntry = (PMMFREE_POOL_ENTRY)((ULONG_PTR)FreeEntry + PAGE_SIZE);
FreeEntry->Owner = FirstEntry;
FreeEntry->Signature = MM_FREE_POOL_SIGNATURE;
}
//
// Validate and remember first allocated pool page
//
PointerPte = MiAddressToPte(MmNonPagedPoolStart);
ASSERT(PointerPte->u.Hard.Valid == 1);
MiStartOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);
//
// Keep track of where initial nonpaged pool ends
//
MmNonPagedPoolEnd0 = (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
MmSizeOfNonPagedPoolInBytes);
//
// Validate and remember last allocated pool page
//
PointerPte = MiAddressToPte((PVOID)((ULONG_PTR)MmNonPagedPoolEnd0 - 1));
ASSERT(PointerPte->u.Hard.Valid == 1);
MiEndOfInitialPoolFrame = PFN_FROM_PTE(PointerPte);
//
// Validate the first nonpaged pool expansion page (which is a guard page)
//
PointerPte = MiAddressToPte(MmNonPagedPoolExpansionStart);
ASSERT(PointerPte->u.Hard.Valid == 0);
//
// Calculate the size of the expansion region alone
//
MiExpansionPoolPagesInitialCharge =
BYTES_TO_PAGES(MmMaximumNonPagedPoolInBytes - MmSizeOfNonPagedPoolInBytes);
//
// Remove 2 pages, since there's a guard page on top and on the bottom
//
MiExpansionPoolPagesInitialCharge -= 2;
//
// Now initialize the nonpaged pool expansion PTE space. Remember there's a
// guard page on top so make sure to skip it. The bottom guard page will be
// guaranteed by the fact our size is off by one.
//
MiInitializeSystemPtes(PointerPte + 1,
MiExpansionPoolPagesInitialCharge,
NonPagedPoolExpansion);
}
PVOID
NTAPI
MiMapPageInHyperSpace(IN PEPROCESS Process,
IN PFN_NUMBER Page,
IN PKIRQL OldIrql)
{
MMPTE TempPte;
PMMPTE PointerPte;
PFN_NUMBER Offset;
//
// Never accept page 0 or non-physical pages
//
ASSERT(Page != 0);
ASSERT(MiGetPfnEntry(Page) != NULL);
//
// Build the PTE
//
TempPte = ValidKernelPteLocal;
TempPte.u.Hard.PageFrameNumber = Page;
//
// Pick the first hyperspace PTE
//
PointerPte = MmFirstReservedMappingPte;
//
// Acquire the hyperlock
//
ASSERT(Process == PsGetCurrentProcess());
KeAcquireSpinLock(&Process->HyperSpaceLock, OldIrql);
//
// Now get the first free PTE
//
Offset = PFN_FROM_PTE(PointerPte);
if (!Offset)
{
//
// Reset the PTEs
//
Offset = MI_HYPERSPACE_PTES;
KeFlushProcessTb();
}
//
// Prepare the next PTE
//
PointerPte->u.Hard.PageFrameNumber = Offset - 1;
//
// Write the current PTE
//
PointerPte += Offset;
MI_WRITE_VALID_PTE(PointerPte, TempPte);
//
// Return the address
//
return MiPteToAddress(PointerPte);
}
PVOID
NTAPI
MiMapPagesInZeroSpace(IN PMMPFN Pfn1,
IN PFN_NUMBER NumberOfPages)
{
MMPTE TempPte;
PMMPTE PointerPte;
PFN_NUMBER Offset, PageFrameIndex;
//
// Sanity checks
//
ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
ASSERT(NumberOfPages != 0);
ASSERT(NumberOfPages <= (MI_ZERO_PTES - 1));
//
// Pick the first zeroing PTE
//
PointerPte = MiFirstReservedZeroingPte;
//
// Now get the first free PTE
//
Offset = PFN_FROM_PTE(PointerPte);
if (NumberOfPages > Offset)
{
//
// Reset the PTEs
//
Offset = MI_ZERO_PTES - 1;
PointerPte->u.Hard.PageFrameNumber = Offset;
KeFlushProcessTb();
}
//
// Prepare the next PTE
//
PointerPte->u.Hard.PageFrameNumber = Offset - NumberOfPages;
/* Choose the correct PTE to use, and which template */
PointerPte += (Offset + 1);
TempPte = ValidKernelPte;
/* Make sure the list isn't empty and loop it */
ASSERT(Pfn1 != (PVOID)LIST_HEAD);
while (Pfn1 != (PVOID)LIST_HEAD)
{
/* Get the page index for this PFN */
PageFrameIndex = MiGetPfnEntryIndex(Pfn1);
//
// Write the PFN
//
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
//
// Set the correct PTE to write to, and set its new value
//
PointerPte--;
MI_WRITE_VALID_PTE(PointerPte, TempPte);
/* Move to the next PFN */
Pfn1 = (PMMPFN)Pfn1->u1.Flink;
}
//
// Return the address
//
return MiPteToAddress(PointerPte);
}