NTSTATUS
MiCcPutPagesInTransition (
IN PMI_READ_INFO MiReadInfo
)
/*++
Routine Description:
This routine allocates physical memory for the specified read-list and
puts all the pages in transition (so collided faults from other threads
for these same pages remain coherent). I/O for any pages not already
resident are issued here. The caller must wait for their completion.
Arguments:
MiReadInfo - Supplies a pointer to the read-list.
Return Value:
STATUS_SUCCESS - all the pages were already resident, reference counts
have been applied and no I/O needs to be waited for.
STATUS_ISSUE_PAGING_IO - the I/O has been issued and the caller must wait.
Various other failure status values indicate the operation failed.
Environment:
Kernel mode. PASSIVE_LEVEL.
--*/
{
NTSTATUS status;
PMMPTE LocalPrototypePte;
PVOID StartingVa;
PFN_NUMBER MdlPages;
KIRQL OldIrql;
MMPTE PteContents;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER ResidentAvailableCharge;
PPFN_NUMBER IoPage;
PPFN_NUMBER ApiPage;
PPFN_NUMBER Page;
PPFN_NUMBER DestinationPage;
ULONG PageColor;
PMMPTE PointerPte;
PMMPTE *ProtoPteArray;
PMMPTE *EndProtoPteArray;
PFN_NUMBER DummyPage;
PMDL Mdl;
PMDL FreeMdl;
PMMPFN PfnProto;
PMMPFN Pfn1;
PMMPFN DummyPfn1;
ULONG i;
PFN_NUMBER DummyTrim;
ULONG NumberOfPagesNeedingIo;
MMPTE TempPte;
PMMPTE PointerPde;
PEPROCESS CurrentProcess;
PMMINPAGE_SUPPORT InPageSupport;
PKPRCB Prcb;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
MiReadInfo->DummyPagePfn = NULL;
FreeMdl = NULL;
CurrentProcess = PsGetCurrentProcess();
PfnProto = NULL;
PointerPde = NULL;
InPageSupport = MiReadInfo->InPageSupport;
Mdl = MI_EXTRACT_PREFETCH_MDL (InPageSupport);
ASSERT (Mdl == MiReadInfo->IoMdl);
IoPage = (PPFN_NUMBER)(Mdl + 1);
ApiPage = (PPFN_NUMBER)(MiReadInfo->ApiMdl + 1);
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
MdlPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES (StartingVa,
Mdl->ByteCount);
if (MdlPages + 1 > MAXUSHORT) {
//
// The PFN ReferenceCount for the dummy page could wrap, refuse the
// request.
//
return STATUS_INSUFFICIENT_RESOURCES;
}
NumberOfPagesNeedingIo = 0;
ProtoPteArray = (PMMPTE *)InPageSupport->BasePte;
EndProtoPteArray = ProtoPteArray + MdlPages;
ASSERT (*ProtoPteArray != NULL);
LOCK_PFN (OldIrql);
//
// Ensure sufficient pages exist for the transfer plus the dummy page.
//
if (((SPFN_NUMBER)MdlPages > (SPFN_NUMBER)(MmAvailablePages - MM_HIGH_LIMIT)) ||
(MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MdlPages)) {
UNLOCK_PFN (OldIrql);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// Charge resident available immediately as the PFN lock may get released
// and reacquired below before all the pages have been locked down.
// Note the dummy page is immediately charged separately.
//
MI_DECREMENT_RESIDENT_AVAILABLE (MdlPages, MM_RESAVAIL_ALLOCATE_BUILDMDL);
ResidentAvailableCharge = MdlPages;
//
// Allocate a dummy page to map discarded pages that aren't skipped.
//
DummyPage = MiRemoveAnyPage (0);
Pfn1 = MI_PFN_ELEMENT (DummyPage);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MiInitializePfnForOtherProcess (DummyPage, MI_PF_DUMMY_PAGE_PTE, 0);
//
// Give the page a containing frame so MiIdentifyPfn won't crash.
//
Pfn1->u4.PteFrame = PsInitialSystemProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT;
//
// Always bias the reference count by 1 and charge for this locked page
// up front so the myriad increments and decrements don't get slowed
// down with needless checking.
//
Pfn1->u3.e1.PrototypePte = 0;
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
Pfn1->u3.e1.ReadInProgress = 1;
MiReadInfo->DummyPagePfn = Pfn1;
DummyPfn1 = Pfn1;
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount + MdlPages);
//
// Properly initialize the inpage support block fields we overloaded.
//
InPageSupport->BasePte = *ProtoPteArray;
//
// Build the proper InPageSupport and MDL to describe this run.
//
for (; ProtoPteArray < EndProtoPteArray; ProtoPteArray += 1, IoPage += 1, ApiPage += 1) {
//
// Fill the MDL entry for this RLE.
//
PointerPte = *ProtoPteArray;
ASSERT (PointerPte != NULL);
//
// The PointerPte better be inside a prototype PTE allocation
// so that subsequent page trims update the correct PTEs.
//
ASSERT (((PointerPte >= (PMMPTE)MmPagedPoolStart) &&
(PointerPte <= (PMMPTE)MmPagedPoolEnd)) ||
((PointerPte >= (PMMPTE)MmSpecialPoolStart) && (PointerPte <= (PMMPTE)MmSpecialPoolEnd)));
//
// Check the state of this prototype PTE now that the PFN lock is held.
// If the page is not resident, the PTE must be put in transition with
// read in progress before the PFN lock is released.
//
//
// Lock page containing prototype PTEs in memory by
// incrementing the reference count for the page.
// Unlock any page locked earlier containing prototype PTEs if
// the containing page is not the same for both.
//
if (PfnProto != NULL) {
if (PointerPde != MiGetPteAddress (PointerPte)) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
PfnProto = NULL;
}
}
if (PfnProto == NULL) {
ASSERT (!MI_IS_PHYSICAL_ADDRESS (PointerPte));
PointerPde = MiGetPteAddress (PointerPte);
if (PointerPde->u.Hard.Valid == 0) {
MiMakeSystemAddressValidPfn (PointerPte, OldIrql);
}
PfnProto = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
MI_ADD_LOCKED_PAGE_CHARGE (PfnProto);
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
}
recheck:
PteContents = *PointerPte;
// LWFIX: are zero or dzero ptes possible here ?
ASSERT (PteContents.u.Long != 0);
if (PteContents.u.Hard.Valid == 1) {
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
*ApiPage = PageFrameIndex;
*IoPage = DummyPage;
continue;
}
if ((PteContents.u.Soft.Prototype == 0) &&
(PteContents.u.Soft.Transition == 1)) {
//
// The page is in transition. If there is an inpage still in
// progress, wait for it to complete. Reference the PFN and
// then march on.
//
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
if (Pfn1->u4.InPageError) {
//
// There was an in-page read error and there are other
// threads colliding for this page, delay to let the
// other threads complete and then retry.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
if (Pfn1->u3.e1.ReadInProgress) {
// LWFIX - start with temp\aw.c
}
//
// PTE refers to a normal transition PTE.
//
ASSERT ((SPFN_NUMBER)MmAvailablePages >= 0);
if (MmAvailablePages == 0) {
//
// This can only happen if the system is utilizing a hardware
// compression cache. This ensures that only a safe amount
// of the compressed virtual cache is directly mapped so that
// if the hardware gets into trouble, we can bail it out.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
//
// The PFN reference count will be 1 already here if the
// modified writer has begun a write of this page. Otherwise
// it's ordinarily 0.
//
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
*IoPage = DummyPage;
*ApiPage = PageFrameIndex;
continue;
}
// LWFIX: need to handle protos that are now pagefile (or dzero)
// backed - prefetching it from the file here would cause us to lose
// the contents. Note this can happen for session-space images
// as we back modified (ie: for relocation fixups or IAT
// updated) portions from the pagefile. remove the assert below too.
ASSERT (PteContents.u.Soft.Prototype == 1);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (NULL, OldIrql))) {
//
// Had to wait so recheck all state.
//
goto recheck;
}
NumberOfPagesNeedingIo += 1;
//
// Allocate a physical page.
//
PageColor = MI_PAGE_COLOR_VA_PROCESS (
MiGetVirtualAddressMappedByPte (PointerPte),
&CurrentProcess->NextPageColor);
PageFrameIndex = MiRemoveAnyPage (PageColor);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (PointerPte->u.Hard.Valid == 0);
//
// Initialize read-in-progress PFN.
//
MiInitializePfn (PageFrameIndex, PointerPte, 0);
//
// These pieces of MiInitializePfn initialization are overridden
// here as these pages are only going into prototype
// transition and not into any page tables.
//
Pfn1->u3.e1.PrototypePte = 1;
Pfn1->u2.ShareCount -= 1;
ASSERT (Pfn1->u2.ShareCount == 0);
Pfn1->u3.e1.PageLocation = ZeroedPageList;
Pfn1->u3.e2.ReferenceCount -= 1;
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
//
// Initialize the I/O specific fields.
//
Pfn1->u1.Event = &InPageSupport->Event;
Pfn1->u3.e1.ReadInProgress = 1;
ASSERT (Pfn1->u4.InPageError == 0);
//
// Increment the PFN reference count in the control area for
// the subsection.
//
MiReadInfo->ControlArea->NumberOfPfnReferences += 1;
//
// Put the prototype PTE into the transition state.
//
MI_MAKE_TRANSITION_PTE (TempPte,
PageFrameIndex,
PointerPte->u.Soft.Protection,
PointerPte);
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
*IoPage = PageFrameIndex;
*ApiPage = PageFrameIndex;
}
//
// If all the pages were resident, dereference the dummy page references
// now and notify our caller that I/O is not necessary.
//
if (NumberOfPagesNeedingIo == 0) {
ASSERT (DummyPfn1->u3.e2.ReferenceCount > MdlPages);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - MdlPages);
//
// Unlock page containing prototype PTEs.
//
if (PfnProto != NULL) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
}
UNLOCK_PFN (OldIrql);
//
// Return the upfront resident available charge as the
// individual charges have all been made at this point.
//
MI_INCREMENT_RESIDENT_AVAILABLE (ResidentAvailableCharge,
MM_RESAVAIL_FREE_BUILDMDL_EXCESS);
return STATUS_SUCCESS;
}
//
// Carefully trim leading dummy pages.
//
Page = (PPFN_NUMBER)(Mdl + 1);
DummyTrim = 0;
for (i = 0; i < MdlPages - 1; i += 1) {
if (*Page == DummyPage) {
DummyTrim += 1;
Page += 1;
}
else {
break;
}
}
if (DummyTrim != 0) {
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
ASSERT (Mdl->ByteCount != 0);
InPageSupport->ReadOffset.QuadPart += (DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
//
// Shuffle down the PFNs in the MDL.
// Recalculate BasePte to adjust for the shuffle.
//
Pfn1 = MI_PFN_ELEMENT (*Page);
ASSERT (Pfn1->PteAddress->u.Hard.Valid == 0);
ASSERT ((Pfn1->PteAddress->u.Soft.Prototype == 0) &&
(Pfn1->PteAddress->u.Soft.Transition == 1));
InPageSupport->BasePte = Pfn1->PteAddress;
DestinationPage = (PPFN_NUMBER)(Mdl + 1);
do {
*DestinationPage = *Page;
DestinationPage += 1;
Page += 1;
i += 1;
} while (i < MdlPages);
MdlPages -= DummyTrim;
}
//
// Carefully trim trailing dummy pages.
//
ASSERT (MdlPages != 0);
Page = (PPFN_NUMBER)(Mdl + 1) + MdlPages - 1;
if (*Page == DummyPage) {
ASSERT (MdlPages >= 2);
//
// Trim the last page specially as it may be a partial page.
//
Mdl->Size -= sizeof(PFN_NUMBER);
if (BYTE_OFFSET(Mdl->ByteCount) != 0) {
Mdl->ByteCount &= ~(PAGE_SIZE - 1);
}
else {
Mdl->ByteCount -= PAGE_SIZE;
}
ASSERT (Mdl->ByteCount != 0);
DummyPfn1->u3.e2.ReferenceCount -= 1;
//
// Now trim any other trailing pages.
//
Page -= 1;
DummyTrim = 0;
while (Page != ((PPFN_NUMBER)(Mdl + 1))) {
if (*Page != DummyPage) {
break;
}
DummyTrim += 1;
Page -= 1;
}
if (DummyTrim != 0) {
ASSERT (Mdl->Size > (USHORT)(DummyTrim * sizeof(PFN_NUMBER)));
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
}
ASSERT (MdlPages > DummyTrim + 1);
MdlPages -= (DummyTrim + 1);
#if DBG
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
ASSERT (MdlPages == ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa,
Mdl->ByteCount));
#endif
}
//
// If the MDL is not already embedded in the inpage block, see if its
// final size qualifies it - if so, embed it now.
//
if ((Mdl != &InPageSupport->Mdl) &&
(Mdl->ByteCount <= (MM_MAXIMUM_READ_CLUSTER_SIZE + 1) * PAGE_SIZE)){
#if DBG
RtlFillMemoryUlong (&InPageSupport->Page[0],
(MM_MAXIMUM_READ_CLUSTER_SIZE+1) * sizeof (PFN_NUMBER),
0xf1f1f1f1);
#endif
RtlCopyMemory (&InPageSupport->Mdl, Mdl, Mdl->Size);
FreeMdl = Mdl;
Mdl = &InPageSupport->Mdl;
ASSERT (((ULONG_PTR)Mdl & (sizeof(QUAD) - 1)) == 0);
InPageSupport->u1.e1.PrefetchMdlHighBits = ((ULONG_PTR)Mdl >> 3);
}
PVOID
NTAPI
MiAllocatePoolPages(IN POOL_TYPE PoolType,
IN SIZE_T SizeInBytes)
{
PFN_NUMBER PageFrameNumber;
PFN_COUNT SizeInPages, PageTableCount;
ULONG i;
KIRQL OldIrql;
PLIST_ENTRY NextEntry, NextHead, LastHead;
PMMPTE PointerPte, StartPte;
PMMPDE PointerPde;
ULONG EndAllocation;
MMPTE TempPte;
MMPDE TempPde;
PMMPFN Pfn1;
PVOID BaseVa, BaseVaStart;
PMMFREE_POOL_ENTRY FreeEntry;
PKSPIN_LOCK_QUEUE LockQueue;
//
// Figure out how big the allocation is in pages
//
SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes);
//
// Check for overflow
//
if (SizeInPages == 0)
{
//
// Fail
//
return NULL;
}
//
// Handle paged pool
//
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
{
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Lock the paged pool mutex
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Find some empty allocation space
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
MmPagedPoolInfo.PagedPoolHint);
if (i == 0xFFFFFFFF)
{
//
// Get the page bit count
//
i = ((SizeInPages - 1) / PTE_COUNT) + 1;
DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages);
//
// Check if there is enougn paged pool expansion space left
//
if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
//
// Check if we'll have to expand past the last PTE we have available
//
if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// We can only support this much then
//
PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool);
PageTableCount = (PFN_COUNT)(PointerPde + 1 -
MmPagedPoolInfo.NextPdeForPagedPoolExpansion);
ASSERT(PageTableCount < i);
i = PageTableCount;
}
else
{
//
// Otherwise, there is plenty of space left for this expansion
//
PageTableCount = i;
}
//
// Get the template PDE we'll use to expand
//
TempPde = ValidKernelPde;
//
// Get the first PTE in expansion space
//
PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
BaseVa = MiPdeToPte(PointerPde);
BaseVaStart = BaseVa;
//
// Lock the PFN database and loop pages
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// It should not already be valid
//
ASSERT(PointerPde->u.Hard.Valid == 0);
/* Request a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
/* On PAE/x64 systems, there's no double-buffering */
ASSERT(FALSE);
#else
//
// Save it into our double-buffered system page directory
//
MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;
/* Initialize the PFN */
MiInitializePfnForOtherProcess(PageFrameNumber,
(PMMPTE)PointerPde,
MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]);
/* Write the actual PDE now */
// MI_WRITE_VALID_PDE(PointerPde, TempPde);
#endif
//
// Move on to the next expansion address
//
PointerPde++;
BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
i--;
} while (i > 0);
//
// Release the PFN database lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// These pages are now available, clear their availablity bits
//
EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
PTE_COUNT;
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
EndAllocation,
PageTableCount * PTE_COUNT);
//
// Update the next expansion location
//
MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount;
//
// Zero out the newly available memory
//
RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE);
//
// Now try consuming the pages again
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
0);
if (i == 0xFFFFFFFF)
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
}
//
// Update the pool hint if the request was just one page
//
if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;
//
// Update the end bitmap so we know the bounds of this allocation when
// the time comes to free it
//
EndAllocation = i + SizeInPages - 1;
RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);
//
// Now we can release the lock (it mainly protects the bitmap)
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// Now figure out where this allocation starts
//
BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));
//
// Flush the TLB
//
KeFlushEntireTb(TRUE, TRUE);
/* Setup a demand-zero writable PTE */
MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
//
// Find the first and last PTE, then loop them all
//
PointerPte = MiAddressToPte(BaseVa);
StartPte = PointerPte + SizeInPages;
do
{
//
// Write the demand zero PTE and keep going
//
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
} while (++PointerPte < StartPte);
//
// Return the allocation address to the caller
//
return BaseVa;
}
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Allocations of less than 4 pages go into their individual buckets
//
i = SizeInPages - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
//
// Loop through all the free page lists based on the page index
//
NextHead = &MmNonPagedPoolFreeListHead[i];
LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
//
// Acquire the nonpaged pool lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
do
{
//
// Now loop through all the free page entries in this given list
//
NextEntry = NextHead->Flink;
while (NextEntry != NextHead)
{
/* Is freed non paged pool enabled */
if (MmProtectFreedNonPagedPool)
{
/* We need to be able to touch this page, unprotect it */
MiUnProtectFreeNonPagedPool(NextEntry, 0);
}
//
// Grab the entry and see if it can handle our allocation
//
FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
if (FreeEntry->Size >= SizeInPages)
{
//
// It does, so consume the pages from here
//
FreeEntry->Size -= SizeInPages;
//
// The allocation will begin in this free page area
//
BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
(FreeEntry->Size << PAGE_SHIFT));
/* Remove the item from the list, depending if pool is protected */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolRemoveEntryList(&FreeEntry->List);
else
RemoveEntryList(&FreeEntry->List);
//
// However, check if its' still got space left
//
if (FreeEntry->Size != 0)
{
/* Check which list to insert this entry into */
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
/* Insert the entry into the free list head, check for prot. pool */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
else
InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
//
// Grab the PTE for this allocation
//
PointerPte = MiAddressToPte(BaseVa);
ASSERT(PointerPte->u.Hard.Valid == 1);
//
// Grab the PFN NextEntry and index
//
Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
//
// Now mark it as the beginning of an allocation
//
ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as special pool if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK)
{
Pfn1->u4.VerifierAllocation = 1;
}
//
// Check if the allocation is larger than one page
//
if (SizeInPages != 1)
{
//
// Navigate to the last PFN entry and PTE
//
PointerPte += SizeInPages - 1;
ASSERT(PointerPte->u.Hard.Valid == 1);
Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
}
//
// Mark this PFN as the last (might be the same as the first)
//
ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Release the nonpaged pool lock, and return the allocation
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
return BaseVa;
}
//
// Try the next free page entry
//
NextEntry = FreeEntry->List.Flink;
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
} while (++NextHead < LastHead);
//
// If we got here, we're out of space.
// Start by releasing the lock
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Allocate some system PTEs
//
StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
PointerPte = StartPte;
if (StartPte == NULL)
{
//
// Ran out of memory
//
DPRINT1("Out of NP Expansion Pool\n");
return NULL;
}
//
// Acquire the pool lock now
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
//
// Lock the PFN database too
//
LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
//
// Loop the pages
//
TempPte = ValidKernelPte;
do
{
/* Allocate a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->PteAddress = PointerPte;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u4.VerifierAllocation = 0;
/* Write the PTE for it */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
MI_WRITE_VALID_PTE(PointerPte++, TempPte);
} while (--SizeInPages > 0);
//
// This is the last page
//
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Get the first page and mark it as such
//
Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as a verifier allocation if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1;
//
// Release the PFN and nonpaged pool lock
//
KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Return the address
//
return MiPteToAddress(StartPte);
}
VOID
MmZeroPageThread (
VOID
)
/*++
Routine Description:
Implements the NT zeroing page thread. This thread runs
at priority zero and removes a page from the free list,
zeroes it, and places it on the zeroed page list.
Arguments:
StartContext - not used.
Return Value:
None.
Environment:
Kernel mode.
--*/
{
PVOID EndVa;
KIRQL OldIrql;
ULONG PageFrame;
PMMPFN Pfn1;
PVOID StartVa;
PKTHREAD Thread;
PVOID ZeroBase;
ULONG i;
//
// Before this becomes the zero page thread, free the kernel
// initialization code.
//
MiFindInitializationCode (&StartVa, &EndVa);
if (StartVa != NULL) {
MiFreeInitializationCode (StartVa, EndVa);
}
//
// The following code sets the current thread's base priority to zero
// and then sets its current priority to zero. This ensures that the
// thread always runs at a priority of zero.
//
Thread = KeGetCurrentThread();
Thread->BasePriority = 0;
KeSetPriorityThread (Thread, 0);
//
// Loop forever zeroing pages.
//
do {
//
// Wait until there are at least MmZeroPageMinimum pages
// on the free list.
//
KeWaitForSingleObject (&MmZeroingPageEvent,
WrFreePage,
KernelMode,
FALSE,
(PLARGE_INTEGER)NULL);
LOCK_PFN_WITH_TRY (OldIrql);
do {
if ((volatile)MmFreePageListHead.Total == 0) {
//
// No pages on the free list at this time, wait for
// some more.
//
MmZeroingPageThreadActive = FALSE;
UNLOCK_PFN (OldIrql);
break;
} else {
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
for (i = 0; i < MM_MAXIMUM_NUMBER_OF_COLORS; i++) {
PageFrame = MmFreePagesByPrimaryColor[FreePageList][i].Flink;
if (PageFrame != MM_EMPTY_LIST) {
break;
}
}
#else //MM_MAXIMUM_NUMBER_OF_COLORS > 1
PageFrame = MmFreePageListHead.Flink;
#endif //MM_MAXIMUM_NUMBER_OF_COLORS > 1
Pfn1 = MI_PFN_ELEMENT(PageFrame);
ASSERT (PageFrame != MM_EMPTY_LIST);
Pfn1 = MI_PFN_ELEMENT(PageFrame);
MiRemoveAnyPage (MI_GET_SECONDARY_COLOR (PageFrame, Pfn1));
//
// Zero the page using the last color used to map the page.
//
#if defined(_X86_)
ZeroBase = MiMapPageToZeroInHyperSpace (PageFrame);
UNLOCK_PFN (OldIrql);
RtlZeroMemory (ZeroBase, PAGE_SIZE);
#elif defined(_PPC_)
UNLOCK_PFN (OldIrql);
KeZeroPage(PageFrame);
#else
ZeroBase = (PVOID)(Pfn1->u3.e1.PageColor << PAGE_SHIFT);
UNLOCK_PFN (OldIrql);
HalZeroPage(ZeroBase, ZeroBase, PageFrame);
#endif //X86
LOCK_PFN_WITH_TRY (OldIrql);
MiInsertPageInList (MmPageLocationList[ZeroedPageList],
PageFrame);
}
} while(TRUE);
} while (TRUE);
}
BOOLEAN
NTAPI
MmCreateProcessAddressSpace(IN ULONG MinWs,
IN PEPROCESS Process,
OUT PULONG_PTR DirectoryTableBase)
{
KIRQL OldIrql;
PFN_NUMBER TableBasePfn, HyperPfn, HyperPdPfn, HyperPtPfn, WorkingSetPfn;
PMMPTE SystemPte;
MMPTE TempPte, PdePte;
ULONG TableIndex;
PMMPTE PageTablePointer;
/* Make sure we don't already have a page directory setup */
ASSERT(Process->Pcb.DirectoryTableBase[0] == 0);
ASSERT(Process->Pcb.DirectoryTableBase[1] == 0);
ASSERT(Process->WorkingSetPage == 0);
/* Choose a process color */
Process->NextPageColor = (USHORT)RtlRandom(&MmProcessColorSeed);
/* Setup the hyperspace lock */
KeInitializeSpinLock(&Process->HyperSpaceLock);
/* Lock PFN database */
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Get a page for the table base and one for hyper space. The PFNs for
these pages will be initialized in MmInitializeProcessAddressSpace,
when we are already attached to the process. */
TableBasePfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process));
HyperPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process));
HyperPdPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process));
HyperPtPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process));
WorkingSetPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process));
/* Release PFN lock */
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
/* Zero pages */ /// FIXME:
MiZeroPhysicalPage(HyperPfn);
MiZeroPhysicalPage(WorkingSetPfn);
/* Set the base directory pointers */
Process->WorkingSetPage = WorkingSetPfn;
DirectoryTableBase[0] = TableBasePfn << PAGE_SHIFT;
DirectoryTableBase[1] = HyperPfn << PAGE_SHIFT;
/* Get a PTE to map the page directory */
SystemPte = MiReserveSystemPtes(1, SystemPteSpace);
ASSERT(SystemPte != NULL);
/* Get its address */
PageTablePointer = MiPteToAddress(SystemPte);
/* Build the PTE for the page directory and map it */
PdePte = ValidKernelPte;
PdePte.u.Hard.PageFrameNumber = TableBasePfn;
*SystemPte = PdePte;
/// architecture specific
//MiInitializePageDirectoryForProcess(
/* Copy the kernel mappings and zero out the rest */
TableIndex = PXE_PER_PAGE / 2;
RtlZeroMemory(PageTablePointer, TableIndex * sizeof(MMPTE));
RtlCopyMemory(PageTablePointer + TableIndex,
MiAddressToPxe(0) + TableIndex,
PAGE_SIZE - TableIndex * sizeof(MMPTE));
/* Sanity check */
ASSERT(MiAddressToPxi(MmHyperSpaceEnd) >= TableIndex);
/* Setup a PTE for the page directory mappings */
TempPte = ValidKernelPte;
/* Update the self mapping of the PML4 */
TableIndex = MiAddressToPxi((PVOID)PXE_SELFMAP);
TempPte.u.Hard.PageFrameNumber = TableBasePfn;
PageTablePointer[TableIndex] = TempPte;
/* Write the PML4 entry for hyperspace */
TableIndex = MiAddressToPxi((PVOID)HYPER_SPACE);
TempPte.u.Hard.PageFrameNumber = HyperPfn;
PageTablePointer[TableIndex] = TempPte;
/* Map the hyperspace PDPT to the system PTE */
PdePte.u.Hard.PageFrameNumber = HyperPfn;
*SystemPte = PdePte;
__invlpg(PageTablePointer);
/* Write the hyperspace entry for the first PD */
TempPte.u.Hard.PageFrameNumber = HyperPdPfn;
PageTablePointer[0] = TempPte;
/* Map the hyperspace PD to the system PTE */
PdePte.u.Hard.PageFrameNumber = HyperPdPfn;
*SystemPte = PdePte;
__invlpg(PageTablePointer);
/* Write the hyperspace entry for the first PT */
TempPte.u.Hard.PageFrameNumber = HyperPtPfn;
PageTablePointer[0] = TempPte;
/* Map the hyperspace PT to the system PTE */
PdePte.u.Hard.PageFrameNumber = HyperPtPfn;
*SystemPte = PdePte;
__invlpg(PageTablePointer);
/* Write the hyperspace PTE for the working set list index */
TempPte.u.Hard.PageFrameNumber = WorkingSetPfn;
TableIndex = MiAddressToPti(MmWorkingSetList);
PageTablePointer[TableIndex] = TempPte;
/// end architecture specific
/* Release the system PTE */
MiReleaseSystemPtes(SystemPte, 1, SystemPteSpace);
/* Switch to phase 1 initialization */
ASSERT(Process->AddressSpaceInitialized == 0);
Process->AddressSpaceInitialized = 1;
return TRUE;
}
VOID
NTAPI
MmZeroPageThread(VOID)
{
PKTHREAD Thread = KeGetCurrentThread();
PVOID StartAddress, EndAddress;
PVOID WaitObjects[2];
KIRQL OldIrql;
PVOID ZeroAddress;
PFN_NUMBER PageIndex, FreePage;
PMMPFN Pfn1;
/* Get the discardable sections to free them */
MiFindInitializationCode(&StartAddress, &EndAddress);
if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress);
DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed);
/* Set our priority to 0 */
Thread->BasePriority = 0;
KeSetPriorityThread(Thread, 0);
/* Setup the wait objects */
WaitObjects[0] = &MmZeroingPageEvent;
// WaitObjects[1] = &PoSystemIdleTimer; FIXME: Implement idle timer
while (TRUE)
{
KeWaitForMultipleObjects(1, // 2
WaitObjects,
WaitAny,
WrFreePage,
KernelMode,
FALSE,
NULL,
NULL);
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
while (TRUE)
{
if (!MmFreePageListHead.Total)
{
MmZeroingPageThreadActive = FALSE;
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
break;
}
PageIndex = MmFreePageListHead.Flink;
ASSERT(PageIndex != LIST_HEAD);
Pfn1 = MiGetPfnEntry(PageIndex);
MI_SET_USAGE(MI_USAGE_ZERO_LOOP);
MI_SET_PROCESS2("Kernel 0 Loop");
FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex));
/* The first global free page should also be the first on its own list */
if (FreePage != PageIndex)
{
KeBugCheckEx(PFN_LIST_CORRUPT,
0x8F,
FreePage,
PageIndex,
0);
}
Pfn1->u1.Flink = LIST_HEAD;
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
ZeroAddress = MiMapPagesInZeroSpace(Pfn1, 1);
ASSERT(ZeroAddress);
RtlZeroMemory(ZeroAddress, PAGE_SIZE);
MiUnmapPagesInZeroSpace(ZeroAddress, 1);
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
MiInsertPageInList(&MmZeroedPageListHead, PageIndex);
}
}
}
PVOID
NTAPI
MmAllocateSpecialPool(SIZE_T NumberOfBytes, ULONG Tag, POOL_TYPE PoolType, ULONG SpecialType)
{
KIRQL Irql;
MMPTE TempPte = ValidKernelPte;
PMMPTE PointerPte;
PFN_NUMBER PageFrameNumber;
LARGE_INTEGER TickCount;
PVOID Entry;
PPOOL_HEADER Header;
DPRINT1("MmAllocateSpecialPool(%x %x %x %x)\n", NumberOfBytes, Tag, PoolType, SpecialType);
/* Check if the pool is initialized and quit if it's not */
if (!MiSpecialPoolFirstPte) return NULL;
/* Get the pool type */
PoolType &= BASE_POOL_TYPE_MASK;
/* Check whether current IRQL matches the pool type */
Irql = KeGetCurrentIrql();
if (((PoolType == PagedPool) && (Irql > APC_LEVEL)) ||
((PoolType != PagedPool) && (Irql > DISPATCH_LEVEL)))
{
/* Bad caller */
KeBugCheckEx(SPECIAL_POOL_DETECTED_MEMORY_CORRUPTION, Irql, PoolType, NumberOfBytes, 0x30);
}
/* TODO: Take into account various limitations */
/*if ((PoolType != NonPagedPool) &&
MiSpecialPagesNonPaged > MiSpecialPagesNonPagedMaximum)*/
/* Lock PFN database */
Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Reject allocation in case amount of available pages is too small */
if (MmAvailablePages < 0x100)
{
/* Release the PFN database lock */
KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql);
DPRINT1("Special pool: MmAvailablePages 0x%x is too small\n", MmAvailablePages);
return NULL;
}
/* Reject allocation if special pool PTE list is exhausted */
if (MiSpecialPoolFirstPte->u.List.NextEntry == MM_EMPTY_PTE_LIST)
{
/* Release the PFN database lock */
KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql);
DPRINT1("Special pool: No PTEs left!\n");
/* TODO: Expand the special pool */
return NULL;
}
/* Save allocation time */
KeQueryTickCount(&TickCount);
/* Get a pointer to the first PTE */
PointerPte = MiSpecialPoolFirstPte;
/* Set the first PTE pointer to the next one in the list */
MiSpecialPoolFirstPte = MmSystemPteBase + PointerPte->u.List.NextEntry;
/* Allocate a physical page */
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Initialize PFN and make it valid */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
MiInitializePfnAndMakePteValid(PageFrameNumber, PointerPte, TempPte);
/* Release the PFN database lock */
KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql);
/* Put some content into the page. Low value of tick count would do */
Entry = MiPteToAddress(PointerPte);
RtlFillMemory(Entry, PAGE_SIZE, TickCount.LowPart);
/* Calculate header and entry addresses */
if ((SpecialType != 0) &&
((SpecialType == 1) || (!MmSpecialPoolCatchOverruns)))
{
/* We catch underruns. Data is at the beginning of the page */
Header = (PPOOL_HEADER)((PUCHAR)Entry + PAGE_SIZE - sizeof(POOL_HEADER));
}
else
{
/* We catch overruns. Data is at the end of the page */
Header = (PPOOL_HEADER)Entry;
Entry = (PVOID)((ULONG_PTR)((PUCHAR)Entry - NumberOfBytes + PAGE_SIZE) & ~((LONG_PTR)sizeof(POOL_HEADER) - 1));
}
/* Initialize the header */
RtlZeroMemory(Header, sizeof(POOL_HEADER));
/* Save allocation size there */
Header->Ulong1 = (ULONG)NumberOfBytes;
/* Make sure it's all good */
ASSERT((NumberOfBytes <= PAGE_SIZE - sizeof(POOL_HEADER)) &&
(PAGE_SIZE <= 32 * 1024));
/* Mark it as paged or nonpaged */
if (PoolType == PagedPool)
{
/* Add pagedpool flag into the pool header too */
Header->Ulong1 |= SPECIAL_POOL_PAGED;
/* Also mark the next PTE as special-pool-paged */
PointerPte[1].u.Soft.PageFileHigh |= SPECIAL_POOL_PAGED_PTE;
}
else
{
/* Mark the next PTE as special-pool-nonpaged */
PointerPte[1].u.Soft.PageFileHigh |= SPECIAL_POOL_NONPAGED_PTE;
}
/* Finally save tag and put allocation time into the header's blocksize.
That time will be used to check memory consistency within the allocated
page. */
Header->PoolTag = Tag;
Header->BlockSize = (USHORT)TickCount.LowPart;
DPRINT1("%p\n", Entry);
return Entry;
}
LOGICAL
FASTCALL
MiCopyOnWrite (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte
)
/*++
Routine Description:
This routine performs a copy on write operation for the specified
virtual address.
Arguments:
FaultingAddress - Supplies the virtual address which caused the fault.
PointerPte - Supplies the pointer to the PTE which caused the page fault.
Return Value:
Returns TRUE if the page was actually split, FALSE if not.
Environment:
Kernel mode, APCs disabled, working set mutex held.
--*/
{
MMPTE TempPte;
MMPTE TempPte2;
PMMPTE MappingPte;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER NewPageIndex;
PVOID CopyTo;
PVOID CopyFrom;
KIRQL OldIrql;
PMMPFN Pfn1;
PEPROCESS CurrentProcess;
PMMCLONE_BLOCK CloneBlock;
PMMCLONE_DESCRIPTOR CloneDescriptor;
WSLE_NUMBER WorkingSetIndex;
LOGICAL FakeCopyOnWrite;
PMMWSL WorkingSetList;
PVOID SessionSpace;
PLIST_ENTRY NextEntry;
PIMAGE_ENTRY_IN_SESSION Image;
//
// This is called from MmAccessFault, the PointerPte is valid
// and the working set mutex ensures it cannot change state.
//
// Capture the PTE contents to TempPte.
//
TempPte = *PointerPte;
ASSERT (TempPte.u.Hard.Valid == 1);
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Check to see if this is a prototype PTE with copy on write enabled.
//
FakeCopyOnWrite = FALSE;
CurrentProcess = PsGetCurrentProcess ();
CloneBlock = NULL;
if (FaultingAddress >= (PVOID) MmSessionBase) {
WorkingSetList = MmSessionSpace->Vm.VmWorkingSetList;
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
SessionSpace = (PVOID) MmSessionSpace;
MM_SESSION_SPACE_WS_LOCK_ASSERT ();
if (MmSessionSpace->ImageLoadingCount != 0) {
NextEntry = MmSessionSpace->ImageList.Flink;
while (NextEntry != &MmSessionSpace->ImageList) {
Image = CONTAINING_RECORD (NextEntry, IMAGE_ENTRY_IN_SESSION, Link);
if ((FaultingAddress >= Image->Address) &&
(FaultingAddress <= Image->LastAddress)) {
if (Image->ImageLoading) {
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
TempPte.u.Hard.CopyOnWrite = 0;
TempPte.u.Hard.Write = 1;
//
// The page is no longer copy on write, update the PTE
// setting both the dirty bit and the accessed bit.
//
// Even though the page's current backing is the image
// file, the modified writer will convert it to
// pagefile backing when it notices the change later.
//
MI_SET_PTE_DIRTY (TempPte);
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte);
//
// The TB entry must be flushed as the valid PTE with
// the dirty bit clear has been fetched into the TB. If
// it isn't flushed, another fault is generated as the
// dirty bit is not set in the cached TB entry.
//
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
return FALSE;
}
break;
}
NextEntry = NextEntry->Flink;
}
}
}
else {
WorkingSetList = MmWorkingSetList;
SessionSpace = NULL;
//
// If a fork operation is in progress, block until the fork is
// completed, then retry the whole operation as the state of
// everything may have changed between when the mutexes were
// released and reacquired.
//
if (CurrentProcess->ForkInProgress != NULL) {
if (MiWaitForForkToComplete (CurrentProcess) == TRUE) {
return FALSE;
}
}
if (TempPte.u.Hard.CopyOnWrite == 0) {
//
// This is a fork page which is being made private in order
// to change the protection of the page.
// Do not make the page writable.
//
FakeCopyOnWrite = TRUE;
}
}
WorkingSetIndex = MiLocateWsle (FaultingAddress,
WorkingSetList,
Pfn1->u1.WsIndex,
FALSE);
//
// The page must be copied into a new page.
//
LOCK_PFN (OldIrql);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (SessionSpace != NULL ? HYDRA_PROCESS : CurrentProcess, OldIrql))) {
//
// A wait operation was performed to obtain an available
// page and the working set mutex and PFN lock have
// been released and various things may have changed for
// the worse. Rather than examine all the conditions again,
// return and if things are still proper, the fault will
// be taken again.
//
UNLOCK_PFN (OldIrql);
return FALSE;
}
//
// This must be a prototype PTE. Perform the copy on write.
//
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
//
// A page is being copied and made private, the global state of
// the shared page needs to be updated at this point on certain
// hardware. This is done by ORing the dirty bit into the modify bit in
// the PFN element.
//
// Note that a session page cannot be dirty (no POSIX-style forking is
// supported for these drivers).
//
if (SessionSpace != NULL) {
ASSERT ((TempPte.u.Hard.Valid == 1) && (TempPte.u.Hard.Write == 0));
ASSERT (!MI_IS_PTE_DIRTY (TempPte));
NewPageIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_SESSION(MmSessionSpace));
}
else {
MI_CAPTURE_DIRTY_BIT_TO_PFN (PointerPte, Pfn1);
CloneBlock = (PMMCLONE_BLOCK) Pfn1->PteAddress;
//
// Get a new page with the same color as this page.
//
NewPageIndex = MiRemoveAnyPage (
MI_PAGE_COLOR_PTE_PROCESS(PageFrameIndex,
&CurrentProcess->NextPageColor));
}
MiInitializeCopyOnWritePfn (NewPageIndex,
PointerPte,
WorkingSetIndex,
WorkingSetList);
UNLOCK_PFN (OldIrql);
InterlockedIncrement (&KeGetCurrentPrcb ()->MmCopyOnWriteCount);
CopyFrom = PAGE_ALIGN (FaultingAddress);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
NewPageIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
if (Pfn1->u3.e1.CacheAttribute == MiNonCached) {
MI_DISABLE_CACHING (TempPte2);
}
else if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined) {
MI_SET_PTE_WRITE_COMBINE (TempPte2);
}
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
CopyTo = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
CopyTo = MiMapPageInHyperSpace (CurrentProcess,
NewPageIndex,
&OldIrql);
}
KeCopyPage (CopyTo, CopyFrom);
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, CopyTo, OldIrql);
}
if (!FakeCopyOnWrite) {
//
// If the page was really a copy on write page, make it
// accessed, dirty and writable. Also, clear the copy-on-write
// bit in the PTE.
//
MI_SET_PTE_DIRTY (TempPte);
TempPte.u.Hard.Write = 1;
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
TempPte.u.Hard.CopyOnWrite = 0;
}
//
// Regardless of whether the page was really a copy on write,
// the frame field of the PTE must be updated.
//
TempPte.u.Hard.PageFrameNumber = NewPageIndex;
//
// If the modify bit is set in the PFN database for the
// page, the data cache must be flushed. This is due to the
// fact that this process may have been cloned and the cache
// still contains stale data destined for the page we are
// going to remove.
//
ASSERT (TempPte.u.Hard.Valid == 1);
MI_WRITE_VALID_PTE_NEW_PAGE (PointerPte, TempPte);
//
// Flush the TB entry for this page.
//
if (SessionSpace == NULL) {
MI_FLUSH_SINGLE_TB (FaultingAddress, FALSE);
//
// Increment the number of private pages.
//
CurrentProcess->NumberOfPrivatePages += 1;
}
else {
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
}
//
// Decrement the share count for the page which was copied
// as this PTE no longer refers to it.
//
LOCK_PFN (OldIrql);
MiDecrementShareCount (Pfn1, PageFrameIndex);
if (SessionSpace == NULL) {
CloneDescriptor = MiLocateCloneAddress (CurrentProcess,
(PVOID)CloneBlock);
if (CloneDescriptor != NULL) {
//
// Decrement the reference count for the clone block,
// note that this could release and reacquire the mutexes.
//
MiDecrementCloneBlockReference (CloneDescriptor,
CloneBlock,
CurrentProcess,
NULL,
OldIrql);
}
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
