VOID
MiFlushUnusedSections (
VOID
)
/*++
Routine Description:
This routine rumages through the PFN database and attempts
to close any unused sections.
Arguments:
None.
Return Value:
None.
--*/
{
PMMPFN LastPfn;
PMMPFN Pfn1;
PSUBSECTION Subsection;
KIRQL OldIrql;
LOCK_PFN (OldIrql);
Pfn1 = MI_PFN_ELEMENT (MmLowestPhysicalPage + 1);
LastPfn = MI_PFN_ELEMENT(MmHighestPhysicalPage);
while (Pfn1 < LastPfn) {
if (Pfn1->OriginalPte.u.Soft.Prototype == 1) {
if ((Pfn1->u3.e1.PageLocation == ModifiedPageList) ||
(Pfn1->u3.e1.PageLocation == StandbyPageList)) {
//
// Make sure the PTE is not waiting for I/O to complete.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
Subsection = MiGetSubsectionAddress (&Pfn1->OriginalPte);
MiFlushUnusedSectionInternal (Subsection->ControlArea);
}
}
}
Pfn1++;
}
UNLOCK_PFN (OldIrql);
return;
}
BOOLEAN
MiFlushUnusedSectionInternal (
IN PCONTROL_AREA ControlArea
)
{
BOOLEAN result;
KIRQL OldIrql = APC_LEVEL;
if ((ControlArea->NumberOfMappedViews != 0) ||
(ControlArea->NumberOfSectionReferences != 0)) {
//
// The segment is currently in use.
//
return FALSE;
}
//
// The segment has no references, delete it. If the segment
// is already being deleted, set the event field in the control
// area and wait on the event.
//
if ((ControlArea->u.Flags.BeingDeleted) ||
(ControlArea->u.Flags.BeingCreated)) {
return TRUE;
}
//
// Set the being deleted flag and up the number of mapped views
// for the segment. Upping the number of mapped views prevents
// the segment from being deleted and passed to the deletion thread
// while we are forcing a delete.
//
ControlArea->u.Flags.BeingDeleted = 1;
ControlArea->NumberOfMappedViews = 1;
//
// This is a page file backed or image Segment. The Segment is being
// deleted, remove all references to the paging file and physical memory.
//
UNLOCK_PFN (OldIrql);
MiCleanSection (ControlArea);
LOCK_PFN (OldIrql);
return TRUE;
}
VOID
MiDeleteFreeVm (
IN PVOID StartingAddress,
IN PVOID EndingAddress
)
/*++
Routine Description:
Nonpagable routine to call acquire PFN lock and call
MiDeleteVirtualAddresses.
Arguments:
Return Value:
none.
Environment:
Kernel mode, APCs disabled, WorkingSetMutex and AddressCreation mutexes
held.
--*/
{
KIRQL OldIrql;
LOCK_PFN (OldIrql);
//
// Delete the address range.
//
MiDeleteVirtualAddresses (StartingAddress,
EndingAddress,
FALSE,
(PMMVAD)NULL);
UNLOCK_PFN (OldIrql);
}
VOID
MmZeroPageThread (
VOID
)
/*++
Routine Description:
Implements the NT zeroing page thread. This thread runs
at priority zero and removes a page from the free list,
zeroes it, and places it on the zeroed page list.
Arguments:
StartContext - not used.
Return Value:
None.
Environment:
Kernel mode.
--*/
{
PVOID EndVa;
KIRQL OldIrql;
ULONG PageFrame;
PMMPFN Pfn1;
PVOID StartVa;
PKTHREAD Thread;
PVOID ZeroBase;
ULONG i;
//
// Before this becomes the zero page thread, free the kernel
// initialization code.
//
MiFindInitializationCode (&StartVa, &EndVa);
if (StartVa != NULL) {
MiFreeInitializationCode (StartVa, EndVa);
}
//
// The following code sets the current thread's base priority to zero
// and then sets its current priority to zero. This ensures that the
// thread always runs at a priority of zero.
//
Thread = KeGetCurrentThread();
Thread->BasePriority = 0;
KeSetPriorityThread (Thread, 0);
//
// Loop forever zeroing pages.
//
do {
//
// Wait until there are at least MmZeroPageMinimum pages
// on the free list.
//
KeWaitForSingleObject (&MmZeroingPageEvent,
WrFreePage,
KernelMode,
FALSE,
(PLARGE_INTEGER)NULL);
LOCK_PFN_WITH_TRY (OldIrql);
do {
if ((volatile)MmFreePageListHead.Total == 0) {
//
// No pages on the free list at this time, wait for
// some more.
//
MmZeroingPageThreadActive = FALSE;
UNLOCK_PFN (OldIrql);
break;
} else {
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
for (i = 0; i < MM_MAXIMUM_NUMBER_OF_COLORS; i++) {
PageFrame = MmFreePagesByPrimaryColor[FreePageList][i].Flink;
if (PageFrame != MM_EMPTY_LIST) {
break;
}
}
#else //MM_MAXIMUM_NUMBER_OF_COLORS > 1
PageFrame = MmFreePageListHead.Flink;
#endif //MM_MAXIMUM_NUMBER_OF_COLORS > 1
Pfn1 = MI_PFN_ELEMENT(PageFrame);
ASSERT (PageFrame != MM_EMPTY_LIST);
Pfn1 = MI_PFN_ELEMENT(PageFrame);
MiRemoveAnyPage (MI_GET_SECONDARY_COLOR (PageFrame, Pfn1));
//
// Zero the page using the last color used to map the page.
//
#if defined(_X86_)
ZeroBase = MiMapPageToZeroInHyperSpace (PageFrame);
UNLOCK_PFN (OldIrql);
RtlZeroMemory (ZeroBase, PAGE_SIZE);
#elif defined(_PPC_)
UNLOCK_PFN (OldIrql);
KeZeroPage(PageFrame);
#else
ZeroBase = (PVOID)(Pfn1->u3.e1.PageColor << PAGE_SHIFT);
UNLOCK_PFN (OldIrql);
HalZeroPage(ZeroBase, ZeroBase, PageFrame);
#endif //X86
LOCK_PFN_WITH_TRY (OldIrql);
MiInsertPageInList (MmPageLocationList[ZeroedPageList],
PageFrame);
}
} while(TRUE);
} while (TRUE);
}
BOOLEAN
MmCreateProcessAddressSpace (
IN ULONG MinimumWorkingSetSize,
IN PEPROCESS NewProcess,
OUT PULONG_PTR DirectoryTableBase
)
/*++
Routine Description:
This routine creates an address space which maps the system
portion and contains a hyper space entry.
Arguments:
MinimumWorkingSetSize - Supplies the minimum working set size for
this address space. This value is only used
to ensure that ample physical pages exist
to create this process.
NewProcess - Supplies a pointer to the process object being created.
DirectoryTableBase - Returns the value of the newly created
address space's Page Directory (PD) page and
hyper space page.
Return Value:
Returns TRUE if an address space was successfully created, FALSE
if ample physical pages do not exist.
Environment:
Kernel mode. APCs Disabled.
--*/
{
LOGICAL FlushTbNeeded;
PFN_NUMBER PageDirectoryIndex;
PFN_NUMBER HyperSpaceIndex;
PFN_NUMBER PageContainingWorkingSet;
PFN_NUMBER VadBitMapPage;
MMPTE TempPte;
MMPTE TempPte2;
PEPROCESS CurrentProcess;
KIRQL OldIrql;
PMMPFN Pfn1;
ULONG Color;
PMMPTE PointerPte;
ULONG PdeOffset;
PMMPTE MappingPte;
PMMPTE PointerFillPte;
PMMPTE CurrentAddressSpacePde;
//
// Charge commitment for the page directory pages, working set page table
// page, and working set list. If Vad bitmap lookups are enabled, then
// charge for a page or two for that as well.
//
if (MiChargeCommitment (MM_PROCESS_COMMIT_CHARGE, NULL) == FALSE) {
return FALSE;
}
FlushTbNeeded = FALSE;
CurrentProcess = PsGetCurrentProcess ();
NewProcess->NextPageColor = (USHORT) (RtlRandom (&MmProcessColorSeed));
KeInitializeSpinLock (&NewProcess->HyperSpaceLock);
//
// Get the PFN lock to get physical pages.
//
LOCK_PFN (OldIrql);
//
// Check to make sure the physical pages are available.
//
if (MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MinimumWorkingSetSize){
UNLOCK_PFN (OldIrql);
MiReturnCommitment (MM_PROCESS_COMMIT_CHARGE);
//
// Indicate no directory base was allocated.
//
return FALSE;
}
MM_TRACK_COMMIT (MM_DBG_COMMIT_PROCESS_CREATE, MM_PROCESS_COMMIT_CHARGE);
MI_DECREMENT_RESIDENT_AVAILABLE (MinimumWorkingSetSize,
MM_RESAVAIL_ALLOCATE_CREATE_PROCESS);
//
// Allocate a page directory page.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_PTE_PROCESS (PDE_BASE,
&CurrentProcess->NextPageColor);
PageDirectoryIndex = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (PageDirectoryIndex);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Allocate the hyper space page table page.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_PTE_PROCESS (MiGetPdeAddress(HYPER_SPACE),
&CurrentProcess->NextPageColor);
HyperSpaceIndex = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (HyperSpaceIndex);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Remove page(s) for the VAD bitmap.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_VA_PROCESS (MmWorkingSetList,
&CurrentProcess->NextPageColor);
VadBitMapPage = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (VadBitMapPage);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Remove a page for the working set list.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_VA_PROCESS (MmWorkingSetList,
&CurrentProcess->NextPageColor);
PageContainingWorkingSet = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (PageContainingWorkingSet);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
UNLOCK_PFN (OldIrql);
if (FlushTbNeeded == TRUE) {
MI_FLUSH_TB_FOR_CACHED_ATTRIBUTE ();
}
ASSERT (NewProcess->AddressSpaceInitialized == 0);
PS_SET_BITS (&NewProcess->Flags, PS_PROCESS_FLAGS_ADDRESS_SPACE1);
ASSERT (NewProcess->AddressSpaceInitialized == 1);
NewProcess->Vm.MinimumWorkingSetSize = MinimumWorkingSetSize;
NewProcess->WorkingSetPage = PageContainingWorkingSet;
INITIALIZE_DIRECTORY_TABLE_BASE (&DirectoryTableBase[0], PageDirectoryIndex);
INITIALIZE_DIRECTORY_TABLE_BASE (&DirectoryTableBase[1], HyperSpaceIndex);
//
// Initialize the page reserved for hyper space.
//
TempPte = ValidPdePde;
MI_SET_GLOBAL_STATE (TempPte, 0);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
HyperSpaceIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
PointerPte = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
PointerPte = MiMapPageInHyperSpace (CurrentProcess, HyperSpaceIndex, &OldIrql);
}
TempPte.u.Hard.PageFrameNumber = VadBitMapPage;
PointerPte[MiGetPteOffset(VAD_BITMAP_SPACE)] = TempPte;
TempPte.u.Hard.PageFrameNumber = PageContainingWorkingSet;
PointerPte[MiGetPteOffset(MmWorkingSetList)] = TempPte;
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, PointerPte, OldIrql);
}
//
// Set the PTE address in the PFN for the page directory page.
//
Pfn1 = MI_PFN_ELEMENT (PageDirectoryIndex);
Pfn1->PteAddress = (PMMPTE)PDE_BASE;
TempPte = ValidPdePde;
TempPte.u.Hard.PageFrameNumber = HyperSpaceIndex;
MI_SET_GLOBAL_STATE (TempPte, 0);
//
// Add the new process to our internal list prior to filling any
// system PDEs so if a system PDE changes (large page map or unmap)
// it can mark this process for a subsequent update.
//
ASSERT (NewProcess->Pcb.DirectoryTableBase[0] == 0);
LOCK_EXPANSION (OldIrql);
InsertTailList (&MmProcessList, &NewProcess->MmProcessLinks);
UNLOCK_EXPANSION (OldIrql);
//
// Map the page directory page in hyperspace.
//
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
PageDirectoryIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
PointerPte = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
PointerPte = MiMapPageInHyperSpace (CurrentProcess, PageDirectoryIndex, &OldIrql);
}
PdeOffset = MiGetPdeOffset (MmSystemRangeStart);
PointerFillPte = &PointerPte[PdeOffset];
CurrentAddressSpacePde = MiGetPdeAddress (MmSystemRangeStart);
RtlCopyMemory (PointerFillPte,
CurrentAddressSpacePde,
PAGE_SIZE - PdeOffset * sizeof (MMPTE));
//
// Map the working set page table page.
//
PdeOffset = MiGetPdeOffset (HYPER_SPACE);
PointerPte[PdeOffset] = TempPte;
//
// Zero the remaining page directory range used to map the working
// set list and its hash.
//
PdeOffset += 1;
ASSERT (MiGetPdeOffset (MmHyperSpaceEnd) >= PdeOffset);
MiZeroMemoryPte (&PointerPte[PdeOffset],
(MiGetPdeOffset (MmHyperSpaceEnd) - PdeOffset + 1));
//
// Recursively map the page directory page so it points to itself.
//
TempPte.u.Hard.PageFrameNumber = PageDirectoryIndex;
PointerPte[MiGetPdeOffset(PTE_BASE)] = TempPte;
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, PointerPte, OldIrql);
}
InterlockedExchangeAddSizeT (&MmProcessCommit, MM_PROCESS_COMMIT_CHARGE);
//
// Up the session space reference count.
//
MiSessionAddProcess (NewProcess);
return TRUE;
}
NTSTATUS
MmRemovePhysicalMemory (
IN PPHYSICAL_ADDRESS StartAddress,
IN OUT PLARGE_INTEGER NumberOfBytes
)
/*++
Routine Description:
This routine attempts to remove the specified physical address range
from the system.
Arguments:
StartAddress - Supplies the starting physical address.
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
Return Value:
NTSTATUS.
Environment:
Kernel mode. PASSIVE level. No locks held.
--*/
{
ULONG i;
ULONG Additional;
PFN_NUMBER Page;
PFN_NUMBER LastPage;
PFN_NUMBER OriginalLastPage;
PFN_NUMBER start;
PFN_NUMBER PagesReleased;
PMMPFN Pfn1;
PMMPFN StartPfn;
PMMPFN EndPfn;
KIRQL OldIrql;
PFN_NUMBER StartPage;
PFN_NUMBER EndPage;
PFN_COUNT NumberOfPages;
SPFN_NUMBER MaxPages;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER RemovedPages;
LOGICAL Inserted;
NTSTATUS Status;
PMMPTE PointerPte;
PMMPTE EndPte;
PVOID VirtualAddress;
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
PPHYSICAL_MEMORY_RUN NewRun;
LOGICAL PfnDatabaseIsPhysical;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
//
// The system must be configured for dynamic memory addition. This is
// not strictly required to remove the memory, but it's better to check
// for it now under the assumption that the administrator is probably
// going to want to add this range of memory back in - better to give
// the error now and refuse the removal than to refuse the addition
// later.
//
if (MmDynamicPfn == FALSE) {
return STATUS_NOT_SUPPORTED;
}
PfnDatabaseIsPhysical = TRUE;
}
else {
PfnDatabaseIsPhysical = FALSE;
}
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
EndPage = StartPage + NumberOfPages;
if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
//
// Truncate the request into something that can be mapped by the PFN
// database.
//
EndPage = MmHighestPossiblePhysicalPage + 1;
NumberOfPages = (PFN_COUNT)(EndPage - StartPage);
}
//
// The range cannot wrap.
//
if (StartPage >= EndPage) {
return STATUS_INVALID_PARAMETER_1;
}
StartPfn = MI_PFN_ELEMENT (StartPage);
EndPfn = MI_PFN_ELEMENT (EndPage);
ExAcquireFastMutex (&MmDynamicMemoryMutex);
#if DBG
MiDynmemData[0] += 1;
#endif
//
// Decrease all commit limits to reflect the removed memory.
//
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
ASSERT (MmTotalCommitLimit <= MmTotalCommitLimitMaximum);
if ((NumberOfPages + 100 > MmTotalCommitLimit - MmTotalCommittedPages) ||
(MmTotalCommittedPages > MmTotalCommitLimit)) {
#if DBG
MiDynmemData[1] += 1;
#endif
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return STATUS_INSUFFICIENT_RESOURCES;
}
MmTotalCommitLimit -= NumberOfPages;
MmTotalCommitLimitMaximum -= NumberOfPages;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
//
// Check for outstanding promises that cannot be broken.
//
LOCK_PFN (OldIrql);
MaxPages = MI_NONPAGABLE_MEMORY_AVAILABLE() - 100;
if ((SPFN_NUMBER)NumberOfPages > MaxPages) {
#if DBG
MiDynmemData[2] += 1;
#endif
UNLOCK_PFN (OldIrql);
Status = STATUS_INSUFFICIENT_RESOURCES;
goto giveup2;
}
MmResidentAvailablePages -= NumberOfPages;
MmNumberOfPhysicalPages -= NumberOfPages;
//
// The range must be contained in a single entry. It is permissible for
// it to be part of a single entry, but it must not cross multiple entries.
//
Additional = (ULONG)-2;
start = 0;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
if ((StartPage >= Page) && (EndPage <= LastPage)) {
if ((StartPage == Page) && (EndPage == LastPage)) {
Additional = (ULONG)-1;
}
else if ((StartPage == Page) || (EndPage == LastPage)) {
Additional = 0;
}
else {
Additional = 1;
}
break;
}
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
if (Additional == (ULONG)-2) {
#if DBG
MiDynmemData[3] += 1;
#endif
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += NumberOfPages;
UNLOCK_PFN (OldIrql);
Status = STATUS_CONFLICTING_ADDRESSES;
goto giveup2;
}
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
Pfn1->u3.e1.RemovalRequested = 1;
}
//
// The free and zero lists must be pruned now before releasing the PFN
// lock otherwise if another thread allocates the page from these lists,
// the allocation will clear the RemovalRequested flag forever.
//
RemovedPages = MiRemovePhysicalPages (StartPage, EndPage);
if (RemovedPages != NumberOfPages) {
#if DBG
retry:
#endif
Pfn1 = StartPfn;
InterlockedIncrement (&MiDelayPageFaults);
for (i = 0; i < 5; i += 1) {
UNLOCK_PFN (OldIrql);
//
// Attempt to move pages to the standby list. Note that only the
// pages with RemovalRequested set are moved.
//
MiTrimRemovalPagesOnly = TRUE;
MiEmptyAllWorkingSets ();
MiTrimRemovalPagesOnly = FALSE;
MiFlushAllPages ();
KeDelayExecutionThread (KernelMode, FALSE, &MmHalfSecond);
LOCK_PFN (OldIrql);
RemovedPages += MiRemovePhysicalPages (StartPage, EndPage);
if (RemovedPages == NumberOfPages) {
break;
}
//
// RemovedPages doesn't include pages that were freed directly to
// the bad page list via MiDecrementReferenceCount. So use the above
// check purely as an optimization - and walk here when necessary.
//
for ( ; Pfn1 < EndPfn; Pfn1 += 1) {
if (Pfn1->u3.e1.PageLocation != BadPageList) {
break;
}
}
if (Pfn1 == EndPfn) {
RemovedPages = NumberOfPages;
break;
}
}
InterlockedDecrement (&MiDelayPageFaults);
}
if (RemovedPages != NumberOfPages) {
#if DBG
MiDynmemData[4] += 1;
if (MiShowStuckPages != 0) {
RemovedPages = 0;
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
if (Pfn1->u3.e1.PageLocation != BadPageList) {
RemovedPages += 1;
}
}
ASSERT (RemovedPages != 0);
DbgPrint("MmRemovePhysicalMemory : could not get %d of %d pages\n",
RemovedPages, NumberOfPages);
if (MiShowStuckPages & 0x2) {
ULONG PfnsPrinted;
ULONG EnoughShown;
PMMPFN FirstPfn;
PFN_COUNT PfnCount;
PfnCount = 0;
PfnsPrinted = 0;
EnoughShown = 100;
if (MiShowStuckPages & 0x4) {
EnoughShown = (ULONG)-1;
}
DbgPrint("Stuck PFN list: ");
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
if (Pfn1->u3.e1.PageLocation != BadPageList) {
if (PfnCount == 0) {
FirstPfn = Pfn1;
}
PfnCount += 1;
}
else {
if (PfnCount != 0) {
DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
(FirstPfn - MmPfnDatabase) + PfnCount - 1);
PfnsPrinted += 1;
if (PfnsPrinted == EnoughShown) {
break;
}
PfnCount = 0;
}
}
}
if (PfnCount != 0) {
DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
(FirstPfn - MmPfnDatabase) + PfnCount - 1);
}
DbgPrint("\n");
}
if (MiShowStuckPages & 0x8) {
DbgBreakPoint ();
}
if (MiShowStuckPages & 0x10) {
goto retry;
}
}
#endif
UNLOCK_PFN (OldIrql);
Status = STATUS_NO_MEMORY;
goto giveup;
}
#if DBG
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
}
#endif
//
// All the pages in the range have been removed. Update the physical
// memory blocks and other associated housekeeping.
//
if (Additional == 0) {
//
// The range can be split off from an end of an existing chunk so no
// pool growth or shrinkage is required.
//
NewPhysicalMemoryBlock = MmPhysicalMemoryBlock;
OldPhysicalMemoryBlock = NULL;
}
else {
//
// The range cannot be split off from an end of an existing chunk so
// pool growth or shrinkage is required.
//
UNLOCK_PFN (OldIrql);
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + Additional)));
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
i,
' mM');
if (NewPhysicalMemoryBlock == NULL) {
Status = STATUS_INSUFFICIENT_RESOURCES;
#if DBG
MiDynmemData[5] += 1;
#endif
goto giveup;
}
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
RtlZeroMemory (NewPhysicalMemoryBlock, i);
LOCK_PFN (OldIrql);
}
//
// Remove or split the requested range from the existing memory block.
//
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + Additional;
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages - NumberOfPages;
NewRun = &NewPhysicalMemoryBlock->Run[0];
start = 0;
Inserted = FALSE;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
if (Inserted == FALSE) {
if ((StartPage >= Page) && (EndPage <= LastPage)) {
if ((StartPage == Page) && (EndPage == LastPage)) {
ASSERT (Additional == -1);
start += 1;
continue;
}
else if ((StartPage == Page) || (EndPage == LastPage)) {
ASSERT (Additional == 0);
if (StartPage == Page) {
MmPhysicalMemoryBlock->Run[start].BasePage += NumberOfPages;
}
MmPhysicalMemoryBlock->Run[start].PageCount -= NumberOfPages;
}
else {
ASSERT (Additional == 1);
OriginalLastPage = LastPage;
MmPhysicalMemoryBlock->Run[start].PageCount =
StartPage - MmPhysicalMemoryBlock->Run[start].BasePage;
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
NewRun->BasePage = EndPage;
NewRun->PageCount = OriginalLastPage - EndPage;
NewRun += 1;
start += 1;
continue;
}
Inserted = TRUE;
}
}
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Repoint the MmPhysicalMemoryBlock at the new chunk.
// Free the old block after releasing the PFN lock.
//
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
if (EndPage - 1 == MmHighestPhysicalPage) {
MmHighestPhysicalPage = StartPage - 1;
}
//
// Throw away all the removed pages that are currently enqueued.
//
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
MiUnlinkPageFromList (Pfn1);
ASSERT (Pfn1->u1.Flink == 0);
ASSERT (Pfn1->u2.Blink == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
Pfn1->PteAddress = PFN_REMOVED;
Pfn1->u3.e2.ShortFlags = 0;
Pfn1->OriginalPte.u.Long = ZeroKernelPte.u.Long;
Pfn1->PteFrame = 0;
}
//
// Now that the removed pages have been discarded, eliminate the PFN
// entries that mapped them. Straddling entries left over from an
// adjacent earlier removal are not collapsed at this point.
//
//
PagesReleased = 0;
if (PfnDatabaseIsPhysical == FALSE) {
VirtualAddress = (PVOID)ROUND_TO_PAGES(MI_PFN_ELEMENT(StartPage));
PointerPte = MiGetPteAddress (VirtualAddress);
EndPte = MiGetPteAddress (PAGE_ALIGN(MI_PFN_ELEMENT(EndPage)));
while (PointerPte < EndPte) {
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u2.ShareCount == 1);
ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
Pfn1->u2.ShareCount = 0;
MI_SET_PFN_DELETED (Pfn1);
#if DBG
Pfn1->u3.e1.PageLocation = StandbyPageList;
#endif //DBG
MiDecrementReferenceCount (PageFrameIndex);
KeFlushSingleTb (VirtualAddress,
TRUE,
TRUE,
(PHARDWARE_PTE)PointerPte,
ZeroKernelPte.u.Flush);
PagesReleased += 1;
PointerPte += 1;
VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE);
}
MmResidentAvailablePages += PagesReleased;
}
#if DBG
MiDynmemData[6] += 1;
#endif
UNLOCK_PFN (OldIrql);
if (PagesReleased != 0) {
MiReturnCommitment (PagesReleased);
}
ExReleaseFastMutex (&MmDynamicMemoryMutex);
if (OldPhysicalMemoryBlock != NULL) {
ExFreePool (OldPhysicalMemoryBlock);
}
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
return STATUS_SUCCESS;
giveup:
//
// All the pages in the range were not obtained. Back everything out.
//
PageFrameIndex = StartPage;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
LOCK_PFN (OldIrql);
while (PageFrameIndex < EndPage) {
ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
Pfn1->u3.e1.RemovalRequested = 0;
if ((Pfn1->u3.e1.PageLocation == BadPageList) &&
(Pfn1->u3.e1.ParityError == 0)) {
MiUnlinkPageFromList (Pfn1);
MiInsertPageInList (MmPageLocationList[FreePageList],
PageFrameIndex);
}
Pfn1 += 1;
PageFrameIndex += 1;
}
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += NumberOfPages;
UNLOCK_PFN (OldIrql);
giveup2:
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
MmTotalCommitLimit += NumberOfPages;
MmTotalCommitLimitMaximum += NumberOfPages;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return Status;
}
NTSTATUS
MmAddPhysicalMemory (
IN PPHYSICAL_ADDRESS StartAddress,
IN OUT PLARGE_INTEGER NumberOfBytes
)
/*++
Routine Description:
This routine adds the specified physical address range to the system.
This includes initializing PFN database entries and adding it to the
freelists.
Arguments:
StartAddress - Supplies the starting physical address.
NumberOfBytes - Supplies a pointer to the number of bytes being added.
If any bytes were added (ie: STATUS_SUCCESS is being
returned), the actual amount is returned here.
Return Value:
NTSTATUS.
Environment:
Kernel mode. PASSIVE level. No locks held.
--*/
{
ULONG i;
PMMPFN Pfn1;
KIRQL OldIrql;
LOGICAL Inserted;
LOGICAL Updated;
MMPTE TempPte;
PMMPTE PointerPte;
PMMPTE LastPte;
PFN_NUMBER NumberOfPages;
PFN_NUMBER start;
PFN_NUMBER count;
PFN_NUMBER StartPage;
PFN_NUMBER EndPage;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER Page;
PFN_NUMBER LastPage;
PFN_COUNT PagesNeeded;
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
PPHYSICAL_MEMORY_RUN NewRun;
LOGICAL PfnDatabaseIsPhysical;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
//
// The system must be configured for dynamic memory addition. This is
// critical as only then is the database guaranteed to be non-sparse.
//
if (MmDynamicPfn == FALSE) {
return STATUS_NOT_SUPPORTED;
}
PfnDatabaseIsPhysical = TRUE;
}
else {
PfnDatabaseIsPhysical = FALSE;
}
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
EndPage = StartPage + NumberOfPages;
if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
//
// Truncate the request into something that can be mapped by the PFN
// database.
//
EndPage = MmHighestPossiblePhysicalPage + 1;
NumberOfPages = EndPage - StartPage;
}
//
// The range cannot wrap.
//
if (StartPage >= EndPage) {
return STATUS_INVALID_PARAMETER_1;
}
ExAcquireFastMutex (&MmDynamicMemoryMutex);
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
i,
' mM');
if (NewPhysicalMemoryBlock == NULL) {
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// The range cannot overlap any ranges that are already present.
//
start = 0;
LOCK_PFN (OldIrql);
do {
count = MmPhysicalMemoryBlock->Run[start].PageCount;
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
if (count != 0) {
LastPage = Page + count;
if ((StartPage < Page) && (EndPage > Page)) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_CONFLICTING_ADDRESSES;
}
if ((StartPage >= Page) && (StartPage < LastPage)) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_CONFLICTING_ADDRESSES;
}
}
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Fill any gaps in the (sparse) PFN database needed for these pages,
// unless the PFN database was physically allocated and completely
// committed up front.
//
PagesNeeded = 0;
if (PfnDatabaseIsPhysical == FALSE) {
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
while (PointerPte <= LastPte) {
if (PointerPte->u.Hard.Valid == 0) {
PagesNeeded += 1;
}
PointerPte += 1;
}
if (MmAvailablePages < PagesNeeded) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_INSUFFICIENT_RESOURCES;
}
TempPte = ValidKernelPte;
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
while (PointerPte <= LastPte) {
if (PointerPte->u.Hard.Valid == 0) {
PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
MiInitializePfn (PageFrameIndex, PointerPte, 0);
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
*PointerPte = TempPte;
}
PointerPte += 1;
}
MmResidentAvailablePages -= PagesNeeded;
}
//
// If the new range is adjacent to an existing range, just merge it into
// the old block. Otherwise use the new block as a new entry will have to
// be used.
//
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;
NewRun = &NewPhysicalMemoryBlock->Run[0];
start = 0;
Inserted = FALSE;
Updated = FALSE;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
count = MmPhysicalMemoryBlock->Run[start].PageCount;
if (Inserted == FALSE) {
//
// Note overlaps into adjacent ranges were already checked above.
//
if (StartPage == Page + count) {
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
//
// Coalesce below and above to avoid leaving zero length gaps
// as these gaps would prevent callers from removing ranges
// the span them.
//
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
start += 1;
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
count = MmPhysicalMemoryBlock->Run[start].PageCount;
if (StartPage + NumberOfPages == Page) {
MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
count;
MmPhysicalMemoryBlock->NumberOfRuns -= 1;
//
// Copy any remaining entries.
//
if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
&MmPhysicalMemoryBlock->Run[start + 1],
(MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
}
}
}
Updated = TRUE;
break;
}
if (StartPage + NumberOfPages == Page) {
MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
Updated = TRUE;
break;
}
if (StartPage + NumberOfPages <= Page) {
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
//
// Don't insert here - the new entry really belongs
// (at least) one entry further down.
//
continue;
}
}
NewRun->BasePage = StartPage;
NewRun->PageCount = NumberOfPages;
NewRun += 1;
Inserted = TRUE;
Updated = TRUE;
}
}
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// If the memory block has not been updated, then the new entry must
// be added at the very end.
//
if (Updated == FALSE) {
ASSERT (Inserted == FALSE);
NewRun->BasePage = StartPage;
NewRun->PageCount = NumberOfPages;
Inserted = TRUE;
}
//
// Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
// releasing the PFN lock.
//
if (Inserted == TRUE) {
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
}
//
// Note that the page directory (page parent entries on Win64) must be
// filled in at system boot so that already-created processes do not fault
// when referencing the new PFNs.
//
//
// Walk through the memory descriptors and add pages to the
// free list in the PFN database.
//
PageFrameIndex = StartPage;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
if (EndPage - 1 > MmHighestPhysicalPage) {
MmHighestPhysicalPage = EndPage - 1;
}
while (PageFrameIndex < EndPage) {
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ShortFlags == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
ASSERT (Pfn1->PteFrame == 0);
ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
(Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));
//
// Set the PTE address to the physical page for
// virtual address alignment checking.
//
Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);
MiInsertPageInList (MmPageLocationList[FreePageList],
PageFrameIndex);
PageFrameIndex += 1;
Pfn1 += 1;
}
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;
UNLOCK_PFN (OldIrql);
//
// Increase all commit limits to reflect the additional memory.
//
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
MmTotalCommitLimit += NumberOfPages;
MmTotalCommitLimitMaximum += NumberOfPages;
MmTotalCommittedPages += PagesNeeded;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (OldPhysicalMemoryBlock);
//
// Indicate number of bytes actually added to our caller.
//
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
return STATUS_SUCCESS;
}
PPHYSICAL_MEMORY_RANGE
MmGetPhysicalMemoryRanges (
VOID
)
/*++
Routine Description:
This routine returns the virtual address of a nonpaged pool block which
contains the physical memory ranges in the system.
The returned block contains physical address and page count pairs.
The last entry contains zero for both.
The caller must understand that this block can change at any point before
or after this snapshot.
It is the caller's responsibility to free this block.
Arguments:
None.
Return Value:
NULL on failure.
Environment:
Kernel mode. PASSIVE level. No locks held.
--*/
{
ULONG i;
KIRQL OldIrql;
PPHYSICAL_MEMORY_RANGE p;
PPHYSICAL_MEMORY_RANGE PhysicalMemoryBlock;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
ExAcquireFastMutex (&MmDynamicMemoryMutex);
i = sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1);
PhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
i,
'hPmM');
if (PhysicalMemoryBlock == NULL) {
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return NULL;
}
p = PhysicalMemoryBlock;
LOCK_PFN (OldIrql);
ASSERT (i == (sizeof(PHYSICAL_MEMORY_RANGE) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
for (i = 0; i < MmPhysicalMemoryBlock->NumberOfRuns; i += 1) {
p->BaseAddress.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].BasePage * PAGE_SIZE;
p->NumberOfBytes.QuadPart = (LONGLONG)MmPhysicalMemoryBlock->Run[i].PageCount * PAGE_SIZE;
p += 1;
}
p->BaseAddress.QuadPart = 0;
p->NumberOfBytes.QuadPart = 0;
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return PhysicalMemoryBlock;
}
VOID
MiDeleteVirtualAddresses (
IN PUCHAR StartingAddress,
IN PUCHAR EndingAddress,
IN ULONG AddressSpaceDeletion,
IN PMMVAD Vad
)
/*++
Routine Description:
This routine deletes the specified virtual address range within
the current process.
Arguments:
StartingAddress - Supplies the first virtual address to delete.
EndingAddress - Supplies the last address to delete.
AddressSpaceDeletion - Supplies TRUE if the address space is being
deleted, FALSE otherwise. If TRUE is specified
the TB is not flushed and valid addresses are
not removed from the working set.
Vad - Supplies the virtual address descriptor which maps this range
or NULL if we are not concerned about views. From the Vad the
range of prototype PTEs is determined and this information is
used to uncover if the PTE refers to a prototype PTE or a
fork PTE.
Return Value:
None.
Environment:
Kernel mode, called with APCs disabled working set mutex and PFN lock
held. These mutexes may be released and reacquired to fault pages in.
--*/
{
PUCHAR Va;
PVOID TempVa;
PMMPTE PointerPte;
PMMPTE PointerPde;
PMMPTE OriginalPointerPte;
PMMPTE ProtoPte;
PMMPTE LastProtoPte;
PEPROCESS CurrentProcess;
ULONG FlushTb = FALSE;
PSUBSECTION Subsection;
PUSHORT UsedPageTableCount;
KIRQL OldIrql = APC_LEVEL;
MMPTE_FLUSH_LIST FlushList;
FlushList.Count = 0;
MM_PFN_LOCK_ASSERT();
CurrentProcess = PsGetCurrentProcess();
Va = StartingAddress;
PointerPde = MiGetPdeAddress (Va);
PointerPte = MiGetPteAddress (Va);
OriginalPointerPte = PointerPte;
UsedPageTableCount =
&MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Va)];
while (MiDoesPdeExistAndMakeValid (PointerPde,
CurrentProcess,
TRUE) == FALSE) {
//
// This page directory entry is empty, go to the next one.
//
PointerPde += 1;
PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
Va = MiGetVirtualAddressMappedByPte (PointerPte);
if (Va > EndingAddress) {
//
// All done, return.
//
return;
}
UsedPageTableCount += 1;
}
//
// A valid PDE has been located, examine each PTE and delete them.
//
if ((Vad == (PMMVAD)NULL) ||
(Vad->u.VadFlags.PrivateMemory) ||
(Vad->FirstPrototypePte == (PMMPTE)NULL)) {
ProtoPte = (PMMPTE)NULL;
LastProtoPte = (PMMPTE)NULL;
} else {
ProtoPte = Vad->FirstPrototypePte;
LastProtoPte = (PMMPTE)4;
}
//
// Examine each PTE within the address range and delete it.
//
while (Va <= EndingAddress) {
if (((ULONG)Va & PAGE_DIRECTORY_MASK) == 0) {
//
// Note, the initial address could be aligned on a 4mb boundary.
//
//
// The virtual address is on a page directory (4mb) boundary,
// check the next PDE for validity and flush PTEs for previous
// page table page.
//
MiFlushPteList (&FlushList, FALSE, ZeroPte);
//
// If all the entries have been eliminated from the previous
// page table page, delete the page table page itself.
//
if ((*UsedPageTableCount == 0) && (PointerPde->u.Long != 0)) {
TempVa = MiGetVirtualAddressMappedByPte(PointerPde);
MiDeletePte (PointerPde,
TempVa,
AddressSpaceDeletion,
CurrentProcess,
NULL,
NULL);
}
//
// Release the PFN lock. This prevents a single thread
// from forcing other high priority threads from being
// blocked while a large address range is deleted. There
// is nothing magic about the instruction within the
// lock and unlock.
//
UNLOCK_PFN (OldIrql);
PointerPde = MiGetPdeAddress (Va);
LOCK_PFN (OldIrql);
UsedPageTableCount =
&MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Va)];
while (MiDoesPdeExistAndMakeValid (
PointerPde, CurrentProcess, TRUE) == FALSE) {
//
// This page directory entry is empty, go to the next one.
//
PointerPde += 1;
PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
Va = MiGetVirtualAddressMappedByPte (PointerPte);
if (Va > EndingAddress) {
//
// All done, return.
//
return;
}
UsedPageTableCount += 1;
if (LastProtoPte != NULL) {
ProtoPte = MiGetProtoPteAddress(Vad,Va);
Subsection = MiLocateSubsection (Vad,Va);
LastProtoPte = &Subsection->SubsectionBase[Subsection->PtesInSubsection];
#if DBG
if (Vad->u.VadFlags.ImageMap != 1) {
if ((ProtoPte < Subsection->SubsectionBase) ||
(ProtoPte >= LastProtoPte)) {
DbgPrint ("bad proto pte %lx va %lx Vad %lx sub %lx\n",
ProtoPte,Va,Vad,Subsection);
DbgBreakPoint();
}
}
#endif //DBG
}
}
}
//
// The PDE is now valid, delete the ptes
//
if (PointerPte->u.Long != 0) {
#ifdef R4000
ASSERT (PointerPte->u.Hard.Global == 0);
#endif
//
// One less used page table entry in this page table page.
//
*UsedPageTableCount -= 1;
ASSERT (*UsedPageTableCount < PTE_PER_PAGE);
if (IS_PTE_NOT_DEMAND_ZERO (*PointerPte)) {
if (LastProtoPte != NULL) {
if (ProtoPte >= LastProtoPte) {
ProtoPte = MiGetProtoPteAddress(Vad,Va);
Subsection = MiLocateSubsection (Vad,Va);
LastProtoPte = &Subsection->SubsectionBase[Subsection->PtesInSubsection];
}
#if DBG
if (Vad->u.VadFlags.ImageMap != 1) {
if ((ProtoPte < Subsection->SubsectionBase) ||
(ProtoPte >= LastProtoPte)) {
DbgPrint ("bad proto pte %lx va %lx Vad %lx sub %lx\n",
ProtoPte,Va,Vad,Subsection);
DbgBreakPoint();
}
}
#endif //DBG
}
MiDeletePte (PointerPte,
(PVOID)Va,
AddressSpaceDeletion,
CurrentProcess,
ProtoPte,
&FlushList);
} else {
*PointerPte = ZeroPte;
}
}
Va = Va + PAGE_SIZE;
PointerPte++;
ProtoPte++;
}
//
// Flush out entries for the last page table page.
//
MiFlushPteList (&FlushList, FALSE, ZeroPte);
//
// If all the entries have been eliminated from the previous
// page table page, delete the page table page itself.
//
if ((*UsedPageTableCount == 0) && (PointerPde->u.Long != 0)) {
TempVa = MiGetVirtualAddressMappedByPte(PointerPde);
MiDeletePte (PointerPde,
TempVa,
AddressSpaceDeletion,
CurrentProcess,
NULL,
NULL);
}
//
// All done, return.
//
return;
}
BOOLEAN
MiShutdownSystem (
VOID
)
/*++
Routine Description:
This function performs the shutdown of memory management. This
is accomplished by writing out all modified pages which are
destined for files other than the paging file.
All processes have already been killed, the registry shutdown and
shutdown IRPs already sent. On return from this phase all mapped
file data must be flushed and the unused segment list emptied.
This releases all the Mm references to file objects, allowing many
drivers (especially the network) to unload.
Arguments:
None.
Return Value:
TRUE if the pages were successfully written, FALSE otherwise.
--*/
{
SIZE_T ImportListSize;
PLOAD_IMPORTS ImportList;
PLOAD_IMPORTS ImportListNonPaged;
PLIST_ENTRY NextEntry;
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
PFN_NUMBER ModifiedPage;
PMMPFN Pfn1;
PSUBSECTION Subsection;
PCONTROL_AREA ControlArea;
PPFN_NUMBER Page;
PFILE_OBJECT FilePointer;
ULONG ConsecutiveFileLockFailures;
PFN_NUMBER MdlHack[(sizeof(MDL)/sizeof(PFN_NUMBER)) + MM_MAXIMUM_WRITE_CLUSTER];
PMDL Mdl;
NTSTATUS Status;
KEVENT IoEvent;
IO_STATUS_BLOCK IoStatus;
KIRQL OldIrql;
LARGE_INTEGER StartingOffset;
ULONG count;
ULONG i;
//
// Don't do this more than once.
//
if (MmSystemShutdown == 0) {
Mdl = (PMDL) MdlHack;
Page = (PPFN_NUMBER)(Mdl + 1);
KeInitializeEvent (&IoEvent, NotificationEvent, FALSE);
MmInitializeMdl (Mdl, NULL, PAGE_SIZE);
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
MmLockPageableSectionByHandle (ExPageLockHandle);
LOCK_PFN (OldIrql);
ModifiedPage = MmModifiedPageListHead.Flink;
while (ModifiedPage != MM_EMPTY_LIST) {
//
// There are modified pages.
//
Pfn1 = MI_PFN_ELEMENT (ModifiedPage);
if (Pfn1->OriginalPte.u.Soft.Prototype == 1) {
//
// This page is destined for a file.
//
Subsection = MiGetSubsectionAddress (&Pfn1->OriginalPte);
ControlArea = Subsection->ControlArea;
if ((!ControlArea->u.Flags.Image) &&
(!ControlArea->u.Flags.NoModifiedWriting)) {
MiUnlinkPageFromList (Pfn1);
//
// Issue the write.
//
MI_SET_MODIFIED (Pfn1, 0, 0x28);
//
// Up the reference count for the physical page as there
// is I/O in progress.
//
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
*Page = ModifiedPage;
ControlArea->NumberOfMappedViews += 1;
ControlArea->NumberOfPfnReferences += 1;
UNLOCK_PFN (OldIrql);
StartingOffset.QuadPart = MiStartingOffset (Subsection,
Pfn1->PteAddress);
Mdl->StartVa = NULL;
ConsecutiveFileLockFailures = 0;
FilePointer = ControlArea->FilePointer;
retry:
KeClearEvent (&IoEvent);
Status = FsRtlAcquireFileForCcFlushEx (FilePointer);
if (NT_SUCCESS(Status)) {
Status = IoSynchronousPageWrite (FilePointer,
Mdl,
&StartingOffset,
&IoEvent,
&IoStatus);
//
// Release the file we acquired.
//
FsRtlReleaseFileForCcFlush (FilePointer);
}
if (!NT_SUCCESS(Status)) {
//
// Only try the request more than once if the
// filesystem said it had a deadlock.
//
if (Status == STATUS_FILE_LOCK_CONFLICT) {
ConsecutiveFileLockFailures += 1;
if (ConsecutiveFileLockFailures < 5) {
KeDelayExecutionThread (KernelMode,
FALSE,
(PLARGE_INTEGER)&MmShortTime);
goto retry;
}
goto wait_complete;
}
//
// Ignore all I/O failures - there is nothing that
// can be done at this point.
//
KeSetEvent (&IoEvent, 0, FALSE);
}
Status = KeWaitForSingleObject (&IoEvent,
WrPageOut,
KernelMode,
FALSE,
(PLARGE_INTEGER)&MmTwentySeconds);
wait_complete:
if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {
MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
}
if (Status == STATUS_TIMEOUT) {
//
// The write did not complete in 20 seconds, assume
// that the file systems are hung and return an
// error.
//
LOCK_PFN (OldIrql);
MI_SET_MODIFIED (Pfn1, 1, 0xF);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (Pfn1);
ControlArea->NumberOfMappedViews -= 1;
ControlArea->NumberOfPfnReferences -= 1;
//
// This routine returns with the PFN lock released!
//
MiCheckControlArea (ControlArea, OldIrql);
MmUnlockPageableImageSection (ExPageLockHandle);
return FALSE;
}
LOCK_PFN (OldIrql);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (Pfn1);
ControlArea->NumberOfMappedViews -= 1;
ControlArea->NumberOfPfnReferences -= 1;
//
// This routine returns with the PFN lock released!
//
MiCheckControlArea (ControlArea, OldIrql);
LOCK_PFN (OldIrql);
//
// Restart scan at the front of the list.
//
ModifiedPage = MmModifiedPageListHead.Flink;
continue;
}
}
ModifiedPage = Pfn1->u1.Flink;
}
UNLOCK_PFN (OldIrql);
//
// Indicate to the modified page writer that the system has
// shutdown.
//
MmSystemShutdown = 1;
//
// Check to see if the paging file should be overwritten.
// Only free blocks are written.
//
if (MmZeroPageFile) {
MiZeroAllPageFiles ();
}
MmUnlockPageableImageSection (ExPageLockHandle);
}
if (PoCleanShutdownEnabled ()) {
//
// Empty the unused segment list.
//
LOCK_PFN (OldIrql);
MmUnusedSegmentForceFree = (ULONG)-1;
KeSetEvent (&MmUnusedSegmentCleanup, 0, FALSE);
//
// Give it 5 seconds to empty otherwise assume the filesystems are
// hung and march on.
//
for (count = 0; count < 500; count += 1) {
if (IsListEmpty(&MmUnusedSegmentList)) {
break;
}
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode,
FALSE,
(PLARGE_INTEGER)&MmShortTime);
LOCK_PFN (OldIrql);
#if DBG
if (count == 400) {
//
// Everything should have been flushed by now. Give the
// filesystem team a chance to debug this on checked builds.
//
ASSERT (FALSE);
}
#endif
//
// Resignal if needed in case more closed file objects triggered
// additional entries.
//
if (MmUnusedSegmentForceFree == 0) {
MmUnusedSegmentForceFree = (ULONG)-1;
KeSetEvent (&MmUnusedSegmentCleanup, 0, FALSE);
}
}
UNLOCK_PFN (OldIrql);
//
// Get rid of any paged pool references as they will be illegal
// by the time MmShutdownSystem is called again since the filesystems
// will have shutdown.
//
KeWaitForSingleObject (&MmSystemLoadLock,
WrVirtualMemory,
KernelMode,
FALSE,
(PLARGE_INTEGER)NULL);
NextEntry = PsLoadedModuleList.Flink;
while (NextEntry != &PsLoadedModuleList) {
DataTableEntry = CONTAINING_RECORD (NextEntry,
KLDR_DATA_TABLE_ENTRY,
InLoadOrderLinks);
ImportList = (PLOAD_IMPORTS)DataTableEntry->LoadedImports;
if ((ImportList != (PVOID)LOADED_AT_BOOT) &&
(ImportList != (PVOID)NO_IMPORTS_USED) &&
(!SINGLE_ENTRY(ImportList))) {
ImportListSize = ImportList->Count * sizeof(PVOID) + sizeof(SIZE_T);
ImportListNonPaged = (PLOAD_IMPORTS) ExAllocatePoolWithTag (NonPagedPool,
ImportListSize,
'TDmM');
if (ImportListNonPaged != NULL) {
RtlCopyMemory (ImportListNonPaged, ImportList, ImportListSize);
ExFreePool (ImportList);
DataTableEntry->LoadedImports = ImportListNonPaged;
}
else {
//
// Don't bother with the clean shutdown at this point.
//
PopShutdownCleanly = FALSE;
break;
}
}
//
// Free the full DLL name as it is pageable.
//
if (DataTableEntry->FullDllName.Buffer != NULL) {
ExFreePool (DataTableEntry->FullDllName.Buffer);
DataTableEntry->FullDllName.Buffer = NULL;
}
NextEntry = NextEntry->Flink;
}
KeReleaseMutant (&MmSystemLoadLock, 1, FALSE, FALSE);
//
// Close all the pagefile handles, note we still have an object
// reference to each keeping the underlying object resident.
// At the end of Phase1 shutdown we'll release those references
// to trigger the storage stack unload. The handle close must be
// done here however as it will reference pageable structures.
//
for (i = 0; i < MmNumberOfPagingFiles; i += 1) {
//
// Free each pagefile name now as it resides in paged pool and
// may need to be inpaged to be freed. Since the paging files
// are going to be shutdown shortly, now is the time to access
// pageable stuff and get rid of it. Zeroing the buffer pointer
// is sufficient as the only accesses to this are from the
// try-except-wrapped GetSystemInformation APIs and all the
// user processes are gone already.
//
ASSERT (MmPagingFile[i]->PageFileName.Buffer != NULL);
ExFreePool (MmPagingFile[i]->PageFileName.Buffer);
MmPagingFile[i]->PageFileName.Buffer = NULL;
ZwClose (MmPagingFile[i]->FileHandle);
}
}
return TRUE;
}
LOGICAL
FASTCALL
MiCopyOnWrite (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte
)
/*++
Routine Description:
This routine performs a copy on write operation for the specified
virtual address.
Arguments:
FaultingAddress - Supplies the virtual address which caused the fault.
PointerPte - Supplies the pointer to the PTE which caused the page fault.
Return Value:
Returns TRUE if the page was actually split, FALSE if not.
Environment:
Kernel mode, APCs disabled, working set mutex held.
--*/
{
MMPTE TempPte;
MMPTE TempPte2;
PMMPTE MappingPte;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER NewPageIndex;
PVOID CopyTo;
PVOID CopyFrom;
KIRQL OldIrql;
PMMPFN Pfn1;
PEPROCESS CurrentProcess;
PMMCLONE_BLOCK CloneBlock;
PMMCLONE_DESCRIPTOR CloneDescriptor;
WSLE_NUMBER WorkingSetIndex;
LOGICAL FakeCopyOnWrite;
PMMWSL WorkingSetList;
PVOID SessionSpace;
PLIST_ENTRY NextEntry;
PIMAGE_ENTRY_IN_SESSION Image;
//
// This is called from MmAccessFault, the PointerPte is valid
// and the working set mutex ensures it cannot change state.
//
// Capture the PTE contents to TempPte.
//
TempPte = *PointerPte;
ASSERT (TempPte.u.Hard.Valid == 1);
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Check to see if this is a prototype PTE with copy on write enabled.
//
FakeCopyOnWrite = FALSE;
CurrentProcess = PsGetCurrentProcess ();
CloneBlock = NULL;
if (FaultingAddress >= (PVOID) MmSessionBase) {
WorkingSetList = MmSessionSpace->Vm.VmWorkingSetList;
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
SessionSpace = (PVOID) MmSessionSpace;
MM_SESSION_SPACE_WS_LOCK_ASSERT ();
if (MmSessionSpace->ImageLoadingCount != 0) {
NextEntry = MmSessionSpace->ImageList.Flink;
while (NextEntry != &MmSessionSpace->ImageList) {
Image = CONTAINING_RECORD (NextEntry, IMAGE_ENTRY_IN_SESSION, Link);
if ((FaultingAddress >= Image->Address) &&
(FaultingAddress <= Image->LastAddress)) {
if (Image->ImageLoading) {
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
TempPte.u.Hard.CopyOnWrite = 0;
TempPte.u.Hard.Write = 1;
//
// The page is no longer copy on write, update the PTE
// setting both the dirty bit and the accessed bit.
//
// Even though the page's current backing is the image
// file, the modified writer will convert it to
// pagefile backing when it notices the change later.
//
MI_SET_PTE_DIRTY (TempPte);
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte);
//
// The TB entry must be flushed as the valid PTE with
// the dirty bit clear has been fetched into the TB. If
// it isn't flushed, another fault is generated as the
// dirty bit is not set in the cached TB entry.
//
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
return FALSE;
}
break;
}
NextEntry = NextEntry->Flink;
}
}
}
else {
WorkingSetList = MmWorkingSetList;
SessionSpace = NULL;
//
// If a fork operation is in progress, block until the fork is
// completed, then retry the whole operation as the state of
// everything may have changed between when the mutexes were
// released and reacquired.
//
if (CurrentProcess->ForkInProgress != NULL) {
if (MiWaitForForkToComplete (CurrentProcess) == TRUE) {
return FALSE;
}
}
if (TempPte.u.Hard.CopyOnWrite == 0) {
//
// This is a fork page which is being made private in order
// to change the protection of the page.
// Do not make the page writable.
//
FakeCopyOnWrite = TRUE;
}
}
WorkingSetIndex = MiLocateWsle (FaultingAddress,
WorkingSetList,
Pfn1->u1.WsIndex,
FALSE);
//
// The page must be copied into a new page.
//
LOCK_PFN (OldIrql);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (SessionSpace != NULL ? HYDRA_PROCESS : CurrentProcess, OldIrql))) {
//
// A wait operation was performed to obtain an available
// page and the working set mutex and PFN lock have
// been released and various things may have changed for
// the worse. Rather than examine all the conditions again,
// return and if things are still proper, the fault will
// be taken again.
//
UNLOCK_PFN (OldIrql);
return FALSE;
}
//
// This must be a prototype PTE. Perform the copy on write.
//
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
//
// A page is being copied and made private, the global state of
// the shared page needs to be updated at this point on certain
// hardware. This is done by ORing the dirty bit into the modify bit in
// the PFN element.
//
// Note that a session page cannot be dirty (no POSIX-style forking is
// supported for these drivers).
//
if (SessionSpace != NULL) {
ASSERT ((TempPte.u.Hard.Valid == 1) && (TempPte.u.Hard.Write == 0));
ASSERT (!MI_IS_PTE_DIRTY (TempPte));
NewPageIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_SESSION(MmSessionSpace));
}
else {
MI_CAPTURE_DIRTY_BIT_TO_PFN (PointerPte, Pfn1);
CloneBlock = (PMMCLONE_BLOCK) Pfn1->PteAddress;
//
// Get a new page with the same color as this page.
//
NewPageIndex = MiRemoveAnyPage (
MI_PAGE_COLOR_PTE_PROCESS(PageFrameIndex,
&CurrentProcess->NextPageColor));
}
MiInitializeCopyOnWritePfn (NewPageIndex,
PointerPte,
WorkingSetIndex,
WorkingSetList);
UNLOCK_PFN (OldIrql);
InterlockedIncrement (&KeGetCurrentPrcb ()->MmCopyOnWriteCount);
CopyFrom = PAGE_ALIGN (FaultingAddress);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
NewPageIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
if (Pfn1->u3.e1.CacheAttribute == MiNonCached) {
MI_DISABLE_CACHING (TempPte2);
}
else if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined) {
MI_SET_PTE_WRITE_COMBINE (TempPte2);
}
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
CopyTo = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
CopyTo = MiMapPageInHyperSpace (CurrentProcess,
NewPageIndex,
&OldIrql);
}
KeCopyPage (CopyTo, CopyFrom);
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, CopyTo, OldIrql);
}
if (!FakeCopyOnWrite) {
//
// If the page was really a copy on write page, make it
// accessed, dirty and writable. Also, clear the copy-on-write
// bit in the PTE.
//
MI_SET_PTE_DIRTY (TempPte);
TempPte.u.Hard.Write = 1;
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
TempPte.u.Hard.CopyOnWrite = 0;
}
//
// Regardless of whether the page was really a copy on write,
// the frame field of the PTE must be updated.
//
TempPte.u.Hard.PageFrameNumber = NewPageIndex;
//
// If the modify bit is set in the PFN database for the
// page, the data cache must be flushed. This is due to the
// fact that this process may have been cloned and the cache
// still contains stale data destined for the page we are
// going to remove.
//
ASSERT (TempPte.u.Hard.Valid == 1);
MI_WRITE_VALID_PTE_NEW_PAGE (PointerPte, TempPte);
//
// Flush the TB entry for this page.
//
if (SessionSpace == NULL) {
MI_FLUSH_SINGLE_TB (FaultingAddress, FALSE);
//
// Increment the number of private pages.
//
CurrentProcess->NumberOfPrivatePages += 1;
}
else {
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
}
//
// Decrement the share count for the page which was copied
// as this PTE no longer refers to it.
//
LOCK_PFN (OldIrql);
MiDecrementShareCount (Pfn1, PageFrameIndex);
if (SessionSpace == NULL) {
CloneDescriptor = MiLocateCloneAddress (CurrentProcess,
(PVOID)CloneBlock);
if (CloneDescriptor != NULL) {
//
// Decrement the reference count for the clone block,
// note that this could release and reacquire the mutexes.
//
MiDecrementCloneBlockReference (CloneDescriptor,
CloneBlock,
CurrentProcess,
NULL,
OldIrql);
}
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
VOID
MiProcessValidPteList (
IN PMMPTE *ValidPteList,
IN ULONG Count
)
/*++
Routine Description:
This routine flushes the specified range of valid PTEs.
Arguments:
ValidPteList - Supplies a pointer to an array of PTEs to flush.
Count - Supplies the count of the number of elements in the array.
Return Value:
none.
Environment:
Kernel mode, APCs disabled, WorkingSetMutex and AddressCreation mutexes
held.
--*/
{
ULONG i;
MMPTE_FLUSH_LIST PteFlushList;
MMPTE PteContents;
PMMPFN Pfn1;
PMMPFN Pfn2;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER PageTableFrameIndex;
KIRQL OldIrql;
i = 0;
PteFlushList.Count = Count;
if (Count < MM_MAXIMUM_FLUSH_COUNT) {
do {
PteFlushList.FlushVa[i] =
MiGetVirtualAddressMappedByPte (ValidPteList[i]);
i += 1;
} while (i != Count);
i = 0;
}
LOCK_PFN (OldIrql);
do {
PteContents = *ValidPteList[i];
ASSERT (PteContents.u.Hard.Valid == 1);
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE(&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Decrement the share and valid counts of the page table
// page which maps this PTE.
//
PageTableFrameIndex = Pfn1->u4.PteFrame;
Pfn2 = MI_PFN_ELEMENT (PageTableFrameIndex);
MiDecrementShareCountInline (Pfn2, PageTableFrameIndex);
MI_SET_PFN_DELETED (Pfn1);
//
// Decrement the share count for the physical page. As the page
// is private it will be put on the free list.
//
MiDecrementShareCount (Pfn1, PageFrameIndex);
MI_WRITE_INVALID_PTE (ValidPteList[i], MmDecommittedPte);
i += 1;
} while (i != Count);
MiFlushPteList (&PteFlushList);
UNLOCK_PFN (OldIrql);
return;
}
ULONG
MiDecommitPages (
IN PVOID StartingAddress,
IN PMMPTE EndingPte,
IN PEPROCESS Process,
IN PMMVAD_SHORT Vad
)
/*++
Routine Description:
This routine decommits the specified range of pages.
Arguments:
StartingAddress - Supplies the starting address of the range.
EndingPte - Supplies the ending PTE of the range.
Process - Supplies the current process.
Vad - Supplies the virtual address descriptor which describes the range.
Return Value:
Value to reduce commitment by for the VAD.
Environment:
Kernel mode, APCs disabled, AddressCreation mutex held.
--*/
{
PMMPTE PointerPde;
PMMPTE PointerPte;
PVOID Va;
ULONG CommitReduction;
PMMPTE CommitLimitPte;
KIRQL OldIrql;
PMMPTE ValidPteList[MM_VALID_PTE_SIZE];
ULONG count;
WSLE_NUMBER WorkingSetIndex;
PMMPFN Pfn1;
PMMPFN Pfn2;
WSLE_NUMBER Entry;
MMWSLENTRY Locked;
MMPTE PteContents;
PFN_NUMBER PageTableFrameIndex;
PVOID UsedPageTableHandle;
PETHREAD CurrentThread;
count = 0;
CommitReduction = 0;
if (Vad->u.VadFlags.MemCommit) {
CommitLimitPte = MiGetPteAddress (MI_VPN_TO_VA (Vad->EndingVpn));
}
else {
CommitLimitPte = NULL;
}
//
// Decommit each page by setting the PTE to be explicitly
// decommitted. The PTEs cannot be deleted all at once as
// this would set the PTEs to zero which would auto-evaluate
// as committed if referenced by another thread when a page
// table page is being in-paged.
//
PointerPde = MiGetPdeAddress (StartingAddress);
PointerPte = MiGetPteAddress (StartingAddress);
Va = StartingAddress;
//
// Loop through all the PDEs which map this region and ensure that
// they exist. If they don't exist create them by touching a
// PTE mapped by the PDE.
//
CurrentThread = PsGetCurrentThread ();
LOCK_WS_UNSAFE (CurrentThread, Process);
MiMakePdeExistAndMakeValid (PointerPde, Process, MM_NOIRQL);
while (PointerPte <= EndingPte) {
if (MiIsPteOnPdeBoundary (PointerPte)) {
PointerPde = MiGetPdeAddress (Va);
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
MiMakePdeExistAndMakeValid (PointerPde, Process, MM_NOIRQL);
}
//
// The working set lock is held. No PTEs can go from
// invalid to valid or valid to invalid. Transition
// PTEs can go from transition to pagefile.
//
PteContents = *PointerPte;
if (PteContents.u.Long != 0) {
if (PointerPte->u.Long == MmDecommittedPte.u.Long) {
//
// This PTE is already decommitted.
//
CommitReduction += 1;
}
else {
Process->NumberOfPrivatePages -= 1;
if (PteContents.u.Hard.Valid == 1) {
//
// Make sure this is not a forked PTE.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
if (Pfn1->u3.e1.PrototypePte) {
//
// MiDeletePte may release both the working set pushlock
// and the PFN lock so the valid PTE list must be
// processed now.
//
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL,
OldIrql);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
else {
//
// PTE is valid, process later when PFN lock is held.
//
if (count == MM_VALID_PTE_SIZE) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
ValidPteList[count] = PointerPte;
count += 1;
//
// Remove address from working set list.
//
WorkingSetIndex = Pfn1->u1.WsIndex;
ASSERT (PAGE_ALIGN(MmWsle[WorkingSetIndex].u1.Long) ==
Va);
//
// Check to see if this entry is locked in the
// working set or locked in memory.
//
Locked = MmWsle[WorkingSetIndex].u1.e1;
MiRemoveWsle (WorkingSetIndex, MmWorkingSetList);
//
// Add this entry to the list of free working set
// entries and adjust the working set count.
//
MiReleaseWsle (WorkingSetIndex, &Process->Vm);
if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {
//
// This entry is locked.
//
MmWorkingSetList->FirstDynamic -= 1;
if (WorkingSetIndex != MmWorkingSetList->FirstDynamic) {
Entry = MmWorkingSetList->FirstDynamic;
ASSERT (MmWsle[Entry].u1.e1.Valid);
MiSwapWslEntries (Entry,
WorkingSetIndex,
&Process->Vm,
FALSE);
}
}
MI_SET_PTE_IN_WORKING_SET (PointerPte, 0);
}
}
else if (PteContents.u.Soft.Prototype) {
//
// This is a forked PTE, just delete it.
//
// MiDeletePte may release both the working set pushlock
// and the PFN lock so the valid PTE list must be
// processed now.
//
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL,
OldIrql);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
else if (PteContents.u.Soft.Transition == 1) {
//
// Transition PTE, get the PFN database lock
// and reprocess this one.
//
LOCK_PFN (OldIrql);
PteContents = *PointerPte;
if (PteContents.u.Soft.Transition == 1) {
//
// PTE is still in transition, delete it.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
MI_SET_PFN_DELETED (Pfn1);
PageTableFrameIndex = Pfn1->u4.PteFrame;
Pfn2 = MI_PFN_ELEMENT (PageTableFrameIndex);
MiDecrementShareCountInline (Pfn2, PageTableFrameIndex);
//
// Check the reference count for the page, if the
// reference count is zero, move the page to the
// free list, if the reference count is not zero,
// ignore this page. When the reference count
// goes to zero, it will be placed on the free list.
//
if (Pfn1->u3.e2.ReferenceCount == 0) {
MiUnlinkPageFromList (Pfn1);
MiReleasePageFileSpace (Pfn1->OriginalPte);
MiInsertPageInFreeList (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE(&PteContents));
}
}
else {
//
// Page MUST be in page file format!
//
ASSERT (PteContents.u.Soft.Valid == 0);
ASSERT (PteContents.u.Soft.Prototype == 0);
ASSERT (PteContents.u.Soft.PageFileHigh != 0);
MiReleasePageFileSpace (PteContents);
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
UNLOCK_PFN (OldIrql);
}
else {
//
// Must be demand zero or paging file format.
//
if (PteContents.u.Soft.PageFileHigh != 0) {
LOCK_PFN (OldIrql);
MiReleasePageFileSpace (PteContents);
UNLOCK_PFN (OldIrql);
}
else {
//
// Don't subtract out the private page count for
// a demand zero page.
//
Process->NumberOfPrivatePages += 1;
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
}
}
else {
//
// The PTE is already zero.
//
//
// Increment the count of non-zero page table entries for this
// page table and the number of private pages for the process.
//
UsedPageTableHandle = MI_GET_USED_PTES_HANDLE (Va);
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableHandle);
if (PointerPte > CommitLimitPte) {
//
// PTE is not committed.
//
CommitReduction += 1;
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
PointerPte += 1;
Va = (PVOID)((PCHAR)Va + PAGE_SIZE);
}
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
}
UNLOCK_WS_UNSAFE (CurrentThread, Process);
return CommitReduction;
}
ULONG
MiDecommitPages (
IN PVOID StartingAddress,
IN PMMPTE EndingPte,
IN PEPROCESS Process,
IN PMMVAD_SHORT Vad
)
/*++
Routine Description:
This routine decommits the specficed range of pages.
Arguments:
StartingAddress - Supplies the starting address of the range.
EndingPte - Supplies the ending PTE of the range.
Process - Supplies the current process.
Vad - Supplies the virtual address descriptor which describes the range.
Return Value:
Value to reduce commitment by for the VAD.
Environment:
Kernel mode, APCs disable, WorkingSetMutex and AddressCreation mutexes
held.
--*/
{
PMMPTE PointerPde;
PMMPTE PointerPte;
PVOID Va;
ULONG PdeOffset;
ULONG CommitReduction = 0;
PMMPTE CommitLimitPte;
KIRQL OldIrql;
PMMPTE ValidPteList[MM_VALID_PTE_SIZE];
ULONG count = 0;
ULONG WorkingSetIndex;
PMMPFN Pfn1;
PMMPFN Pfn2;
PVOID SwapVa;
ULONG Entry;
MMWSLENTRY Locked;
MMPTE PteContents;
if (Vad->u.VadFlags.MemCommit) {
CommitLimitPte = MiGetPteAddress (Vad->EndingVa);
} else {
CommitLimitPte = NULL;
}
//
// Decommit each page by setting the PTE to be explicitly
// decommitted. The PTEs cannot be deleted all at once as
// this would set the PTEs to zero which would auto-evaluate
// as committed if referenced by another thread when a page
// table page is being in-paged.
//
PointerPde = MiGetPdeAddress (StartingAddress);
PointerPte = MiGetPteAddress (StartingAddress);
Va = StartingAddress;
PdeOffset = MiGetPdeOffset (Va);
//
// Loop through all the PDEs which map this region and ensure that
// they exist. If they don't exist create them by touching a
// PTE mapped by the PDE.
//
//
// Get the PFN mutex so the MiDeletePte can be called.
//
MiMakePdeExistAndMakeValid(PointerPde, Process, FALSE);
while (PointerPte <= EndingPte) {
if (((ULONG)PointerPte & (PAGE_SIZE - 1)) == 0) {
PdeOffset = MiGetPdeOffset (Va);
PointerPde = MiGetPdeAddress (Va);
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
MiMakePdeExistAndMakeValid(PointerPde, Process, FALSE);
}
//
// The working set lock is held. No PTEs can go from
// invalid to valid or valid to invalid. Transition
// PTEs can go from transition to pagefile.
//
PteContents = *PointerPte;
if (PteContents.u.Long != 0) {
if (PointerPte->u.Long == MmDecommittedPte.u.Long) {
//
// This PTE is already decommitted.
//
CommitReduction += 1;
} else {
Process->NumberOfPrivatePages -= 1;
if (PteContents.u.Hard.Valid == 1) {
//
// Make sure this is not a forked PTE.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
if (Pfn1->u3.e1.PrototypePte) {
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
*PointerPte = MmDecommittedPte;
} else {
//
// Pte is valid, process later when PFN lock is held.
//
if (count == MM_VALID_PTE_SIZE) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
ValidPteList[count] = PointerPte;
count += 1;
//
// Remove address from working set list.
//
WorkingSetIndex = Pfn1->u1.WsIndex;
ASSERT (PAGE_ALIGN(MmWsle[WorkingSetIndex].u1.Long) ==
Va);
//
// Check to see if this entry is locked in the working set
// or locked in memory.
//
Locked = MmWsle[WorkingSetIndex].u1.e1;
MiRemoveWsle (WorkingSetIndex, MmWorkingSetList);
//
// Add this entry to the list of free working set entries
// and adjust the working set count.
//
MiReleaseWsle (WorkingSetIndex, &Process->Vm);
if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {
//
// This entry is locked.
//
MmWorkingSetList->FirstDynamic -= 1;
if (WorkingSetIndex != MmWorkingSetList->FirstDynamic) {
SwapVa = MmWsle[MmWorkingSetList->FirstDynamic].u1.VirtualAddress;
SwapVa = PAGE_ALIGN (SwapVa);
Pfn2 = MI_PFN_ELEMENT (
MiGetPteAddress (SwapVa)->u.Hard.PageFrameNumber);
Entry = MiLocateWsle (SwapVa,
MmWorkingSetList,
Pfn2->u1.WsIndex);
MiSwapWslEntries (Entry,
WorkingSetIndex,
&Process->Vm);
}
}
}
} else if (PteContents.u.Soft.Prototype) {
//
// This is a forked PTE, just delete it.
//
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
*PointerPte = MmDecommittedPte;
} else if (PteContents.u.Soft.Transition == 1) {
//
// Transition PTE, get the PFN database lock
// and reprocess this one.
//
LOCK_PFN (OldIrql);
PteContents = *PointerPte;
if (PteContents.u.Soft.Transition == 1) {
//
// PTE is still in transition, delete it.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
MI_SET_PFN_DELETED (Pfn1);
MiDecrementShareCount (Pfn1->PteFrame);
//
// Check the reference count for the page, if the
// reference count is zero, move the page to the
// free list, if the reference count is not zero,
// ignore this page. When the refernce count
// goes to zero, it will be placed on the free list.
//
if (Pfn1->u3.e2.ReferenceCount == 0) {
MiUnlinkPageFromList (Pfn1);
MiReleasePageFileSpace (Pfn1->OriginalPte);
MiInsertPageInList (MmPageLocationList[FreePageList],
PteContents.u.Trans.PageFrameNumber);
}
*PointerPte = MmDecommittedPte;
} else {
//
// Page MUST be in page file format!
//
ASSERT (PteContents.u.Soft.Valid == 0);
ASSERT (PteContents.u.Soft.Prototype == 0);
ASSERT (PteContents.u.Soft.PageFileHigh != 0);
MiReleasePageFileSpace (PteContents);
*PointerPte = MmDecommittedPte;
}
UNLOCK_PFN (OldIrql);
} else {
//
// Must be demand zero or paging file format.
//
if (PteContents.u.Soft.PageFileHigh != 0) {
LOCK_PFN (OldIrql);
MiReleasePageFileSpace (PteContents);
UNLOCK_PFN (OldIrql);
} else {
//
// Don't subtract out the private page count for
// a demand zero page.
//
Process->NumberOfPrivatePages += 1;
}
*PointerPte = MmDecommittedPte;
}
}
} else {
//
// The PTE is already zero.
//
//
// Increment the count of non-zero page table entires for this
// page table and the number of private pages for the process.
//
MmWorkingSetList->UsedPageTableEntries[PdeOffset] += 1;
if (PointerPte > CommitLimitPte) {
//
// Pte is not committed.
//
CommitReduction += 1;
}
*PointerPte = MmDecommittedPte;
}
PointerPte += 1;
Va = (PVOID)((ULONG)Va + PAGE_SIZE);
}
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
}
return CommitReduction;
}
VOID
MiZeroPageFile (
IN PVOID Context
)
/*++
Routine Description:
This routine zeroes all inactive pagefile blocks in the specified paging
file.
Arguments:
Context - Supplies the information on which pagefile to zero and a zeroed
page to use for the I/O.
Return Value:
Returns TRUE on success, FALSE on failure.
Environment:
Kernel mode, the caller must lock down PAGELK.
--*/
{
PFN_NUMBER MaxPagesToWrite;
PMMPFN Pfn1;
PPFN_NUMBER Page;
PFN_NUMBER MdlHack[(sizeof(MDL)/sizeof(PFN_NUMBER)) + MM_MAXIMUM_WRITE_CLUSTER];
PMDL Mdl;
NTSTATUS Status;
KEVENT IoEvent;
IO_STATUS_BLOCK IoStatus;
KIRQL OldIrql;
LARGE_INTEGER StartingOffset;
ULONG count;
ULONG i;
PFN_NUMBER first;
ULONG write;
PKEVENT AllDone;
SIZE_T NumberOfBytes;
PMMPAGING_FILE PagingFile;
PFN_NUMBER ZeroedPageFrame;
PMM_ZERO_PAGEFILE_CONTEXT ZeroContext;
ZeroContext = (PMM_ZERO_PAGEFILE_CONTEXT) Context;
PagingFile = ZeroContext->PagingFile;
ZeroedPageFrame = ZeroContext->ZeroedPageFrame;
AllDone = ZeroContext->AllDone;
ExFreePool (Context);
NumberOfBytes = MmModifiedWriteClusterSize << PAGE_SHIFT;
MaxPagesToWrite = NumberOfBytes >> PAGE_SHIFT;
Mdl = (PMDL) MdlHack;
Page = (PPFN_NUMBER)(Mdl + 1);
KeInitializeEvent (&IoEvent, NotificationEvent, FALSE);
MmInitializeMdl (Mdl, NULL, PAGE_SIZE);
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
Mdl->StartVa = NULL;
i = 0;
Page = (PPFN_NUMBER)(Mdl + 1);
for (i = 0; i < MaxPagesToWrite; i += 1) {
*Page = ZeroedPageFrame;
Page += 1;
}
count = 0;
write = FALSE;
SATISFY_OVERZEALOUS_COMPILER (first = 0);
LOCK_PFN (OldIrql);
for (i = 1; i < PagingFile->Size; i += 1) {
if (RtlCheckBit (PagingFile->Bitmap, (ULONG) i) == 0) {
//
// Claim the pagefile location as the modified writer
// may already be scanning.
//
RtlSetBit (PagingFile->Bitmap, (ULONG) i);
if (count == 0) {
first = i;
}
count += 1;
if ((count == MaxPagesToWrite) || (i == PagingFile->Size - 1)) {
write = TRUE;
}
}
else {
if (count != 0) {
//
// Issue a write.
//
write = TRUE;
}
}
if (write) {
UNLOCK_PFN (OldIrql);
StartingOffset.QuadPart = (LONGLONG)first << PAGE_SHIFT;
Mdl->ByteCount = count << PAGE_SHIFT;
KeClearEvent (&IoEvent);
Status = IoSynchronousPageWrite (PagingFile->File,
Mdl,
&StartingOffset,
&IoEvent,
&IoStatus);
//
// Ignore all I/O failures - there is nothing that can
// be done at this point.
//
if (!NT_SUCCESS (Status)) {
KeSetEvent (&IoEvent, 0, FALSE);
}
Status = KeWaitForSingleObject (&IoEvent,
WrPageOut,
KernelMode,
FALSE,
(PLARGE_INTEGER)&MmTwentySeconds);
if (Status == STATUS_TIMEOUT) {
//
// The write did not complete in 20 seconds, assume
// that the file systems are hung and return an error.
//
// Note the zero page (and any MDL system virtual address a
// driver may have created) is leaked because we don't know
// what the filesystem or storage stack might (still) be
// doing to them.
//
Pfn1 = MI_PFN_ELEMENT (ZeroedPageFrame);
LOCK_PFN (OldIrql);
//
// Increment the reference count on the zeroed page to ensure
// it is never freed.
//
InterlockedIncrementPfn ((PSHORT)&Pfn1->u3.e2.ReferenceCount);
RtlClearBits (PagingFile->Bitmap, (ULONG) first, count);
break;
}
if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {
MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
}
write = FALSE;
LOCK_PFN (OldIrql);
RtlClearBits (PagingFile->Bitmap, (ULONG) first, count);
count = 0;
}
}
UNLOCK_PFN (OldIrql);
KeSetEvent (AllDone, 0, FALSE);
return;
}
NTSTATUS
MiCcPutPagesInTransition (
IN PMI_READ_INFO MiReadInfo
)
/*++
Routine Description:
This routine allocates physical memory for the specified read-list and
puts all the pages in transition (so collided faults from other threads
for these same pages remain coherent). I/O for any pages not already
resident are issued here. The caller must wait for their completion.
Arguments:
MiReadInfo - Supplies a pointer to the read-list.
Return Value:
STATUS_SUCCESS - all the pages were already resident, reference counts
have been applied and no I/O needs to be waited for.
STATUS_ISSUE_PAGING_IO - the I/O has been issued and the caller must wait.
Various other failure status values indicate the operation failed.
Environment:
Kernel mode. PASSIVE_LEVEL.
--*/
{
NTSTATUS status;
PMMPTE LocalPrototypePte;
PVOID StartingVa;
PFN_NUMBER MdlPages;
KIRQL OldIrql;
MMPTE PteContents;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER ResidentAvailableCharge;
PPFN_NUMBER IoPage;
PPFN_NUMBER ApiPage;
PPFN_NUMBER Page;
PPFN_NUMBER DestinationPage;
ULONG PageColor;
PMMPTE PointerPte;
PMMPTE *ProtoPteArray;
PMMPTE *EndProtoPteArray;
PFN_NUMBER DummyPage;
PMDL Mdl;
PMDL FreeMdl;
PMMPFN PfnProto;
PMMPFN Pfn1;
PMMPFN DummyPfn1;
ULONG i;
PFN_NUMBER DummyTrim;
ULONG NumberOfPagesNeedingIo;
MMPTE TempPte;
PMMPTE PointerPde;
PEPROCESS CurrentProcess;
PMMINPAGE_SUPPORT InPageSupport;
PKPRCB Prcb;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
MiReadInfo->DummyPagePfn = NULL;
FreeMdl = NULL;
CurrentProcess = PsGetCurrentProcess();
PfnProto = NULL;
PointerPde = NULL;
InPageSupport = MiReadInfo->InPageSupport;
Mdl = MI_EXTRACT_PREFETCH_MDL (InPageSupport);
ASSERT (Mdl == MiReadInfo->IoMdl);
IoPage = (PPFN_NUMBER)(Mdl + 1);
ApiPage = (PPFN_NUMBER)(MiReadInfo->ApiMdl + 1);
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
MdlPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES (StartingVa,
Mdl->ByteCount);
if (MdlPages + 1 > MAXUSHORT) {
//
// The PFN ReferenceCount for the dummy page could wrap, refuse the
// request.
//
return STATUS_INSUFFICIENT_RESOURCES;
}
NumberOfPagesNeedingIo = 0;
ProtoPteArray = (PMMPTE *)InPageSupport->BasePte;
EndProtoPteArray = ProtoPteArray + MdlPages;
ASSERT (*ProtoPteArray != NULL);
LOCK_PFN (OldIrql);
//
// Ensure sufficient pages exist for the transfer plus the dummy page.
//
if (((SPFN_NUMBER)MdlPages > (SPFN_NUMBER)(MmAvailablePages - MM_HIGH_LIMIT)) ||
(MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MdlPages)) {
UNLOCK_PFN (OldIrql);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// Charge resident available immediately as the PFN lock may get released
// and reacquired below before all the pages have been locked down.
// Note the dummy page is immediately charged separately.
//
MI_DECREMENT_RESIDENT_AVAILABLE (MdlPages, MM_RESAVAIL_ALLOCATE_BUILDMDL);
ResidentAvailableCharge = MdlPages;
//
// Allocate a dummy page to map discarded pages that aren't skipped.
//
DummyPage = MiRemoveAnyPage (0);
Pfn1 = MI_PFN_ELEMENT (DummyPage);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MiInitializePfnForOtherProcess (DummyPage, MI_PF_DUMMY_PAGE_PTE, 0);
//
// Give the page a containing frame so MiIdentifyPfn won't crash.
//
Pfn1->u4.PteFrame = PsInitialSystemProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT;
//
// Always bias the reference count by 1 and charge for this locked page
// up front so the myriad increments and decrements don't get slowed
// down with needless checking.
//
Pfn1->u3.e1.PrototypePte = 0;
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
Pfn1->u3.e1.ReadInProgress = 1;
MiReadInfo->DummyPagePfn = Pfn1;
DummyPfn1 = Pfn1;
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount + MdlPages);
//
// Properly initialize the inpage support block fields we overloaded.
//
InPageSupport->BasePte = *ProtoPteArray;
//
// Build the proper InPageSupport and MDL to describe this run.
//
for (; ProtoPteArray < EndProtoPteArray; ProtoPteArray += 1, IoPage += 1, ApiPage += 1) {
//
// Fill the MDL entry for this RLE.
//
PointerPte = *ProtoPteArray;
ASSERT (PointerPte != NULL);
//
// The PointerPte better be inside a prototype PTE allocation
// so that subsequent page trims update the correct PTEs.
//
ASSERT (((PointerPte >= (PMMPTE)MmPagedPoolStart) &&
(PointerPte <= (PMMPTE)MmPagedPoolEnd)) ||
((PointerPte >= (PMMPTE)MmSpecialPoolStart) && (PointerPte <= (PMMPTE)MmSpecialPoolEnd)));
//
// Check the state of this prototype PTE now that the PFN lock is held.
// If the page is not resident, the PTE must be put in transition with
// read in progress before the PFN lock is released.
//
//
// Lock page containing prototype PTEs in memory by
// incrementing the reference count for the page.
// Unlock any page locked earlier containing prototype PTEs if
// the containing page is not the same for both.
//
if (PfnProto != NULL) {
if (PointerPde != MiGetPteAddress (PointerPte)) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
PfnProto = NULL;
}
}
if (PfnProto == NULL) {
ASSERT (!MI_IS_PHYSICAL_ADDRESS (PointerPte));
PointerPde = MiGetPteAddress (PointerPte);
if (PointerPde->u.Hard.Valid == 0) {
MiMakeSystemAddressValidPfn (PointerPte, OldIrql);
}
PfnProto = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
MI_ADD_LOCKED_PAGE_CHARGE (PfnProto);
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
}
recheck:
PteContents = *PointerPte;
// LWFIX: are zero or dzero ptes possible here ?
ASSERT (PteContents.u.Long != 0);
if (PteContents.u.Hard.Valid == 1) {
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
*ApiPage = PageFrameIndex;
*IoPage = DummyPage;
continue;
}
if ((PteContents.u.Soft.Prototype == 0) &&
(PteContents.u.Soft.Transition == 1)) {
//
// The page is in transition. If there is an inpage still in
// progress, wait for it to complete. Reference the PFN and
// then march on.
//
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
if (Pfn1->u4.InPageError) {
//
// There was an in-page read error and there are other
// threads colliding for this page, delay to let the
// other threads complete and then retry.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
if (Pfn1->u3.e1.ReadInProgress) {
// LWFIX - start with temp\aw.c
}
//
// PTE refers to a normal transition PTE.
//
ASSERT ((SPFN_NUMBER)MmAvailablePages >= 0);
if (MmAvailablePages == 0) {
//
// This can only happen if the system is utilizing a hardware
// compression cache. This ensures that only a safe amount
// of the compressed virtual cache is directly mapped so that
// if the hardware gets into trouble, we can bail it out.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
//
// The PFN reference count will be 1 already here if the
// modified writer has begun a write of this page. Otherwise
// it's ordinarily 0.
//
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
*IoPage = DummyPage;
*ApiPage = PageFrameIndex;
continue;
}
// LWFIX: need to handle protos that are now pagefile (or dzero)
// backed - prefetching it from the file here would cause us to lose
// the contents. Note this can happen for session-space images
// as we back modified (ie: for relocation fixups or IAT
// updated) portions from the pagefile. remove the assert below too.
ASSERT (PteContents.u.Soft.Prototype == 1);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (NULL, OldIrql))) {
//
// Had to wait so recheck all state.
//
goto recheck;
}
NumberOfPagesNeedingIo += 1;
//
// Allocate a physical page.
//
PageColor = MI_PAGE_COLOR_VA_PROCESS (
MiGetVirtualAddressMappedByPte (PointerPte),
&CurrentProcess->NextPageColor);
PageFrameIndex = MiRemoveAnyPage (PageColor);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (PointerPte->u.Hard.Valid == 0);
//
// Initialize read-in-progress PFN.
//
MiInitializePfn (PageFrameIndex, PointerPte, 0);
//
// These pieces of MiInitializePfn initialization are overridden
// here as these pages are only going into prototype
// transition and not into any page tables.
//
Pfn1->u3.e1.PrototypePte = 1;
Pfn1->u2.ShareCount -= 1;
ASSERT (Pfn1->u2.ShareCount == 0);
Pfn1->u3.e1.PageLocation = ZeroedPageList;
Pfn1->u3.e2.ReferenceCount -= 1;
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
//
// Initialize the I/O specific fields.
//
Pfn1->u1.Event = &InPageSupport->Event;
Pfn1->u3.e1.ReadInProgress = 1;
ASSERT (Pfn1->u4.InPageError == 0);
//
// Increment the PFN reference count in the control area for
// the subsection.
//
MiReadInfo->ControlArea->NumberOfPfnReferences += 1;
//
// Put the prototype PTE into the transition state.
//
MI_MAKE_TRANSITION_PTE (TempPte,
PageFrameIndex,
PointerPte->u.Soft.Protection,
PointerPte);
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
*IoPage = PageFrameIndex;
*ApiPage = PageFrameIndex;
}
//
// If all the pages were resident, dereference the dummy page references
// now and notify our caller that I/O is not necessary.
//
if (NumberOfPagesNeedingIo == 0) {
ASSERT (DummyPfn1->u3.e2.ReferenceCount > MdlPages);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - MdlPages);
//
// Unlock page containing prototype PTEs.
//
if (PfnProto != NULL) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
}
UNLOCK_PFN (OldIrql);
//
// Return the upfront resident available charge as the
// individual charges have all been made at this point.
//
MI_INCREMENT_RESIDENT_AVAILABLE (ResidentAvailableCharge,
MM_RESAVAIL_FREE_BUILDMDL_EXCESS);
return STATUS_SUCCESS;
}
//
// Carefully trim leading dummy pages.
//
Page = (PPFN_NUMBER)(Mdl + 1);
DummyTrim = 0;
for (i = 0; i < MdlPages - 1; i += 1) {
if (*Page == DummyPage) {
DummyTrim += 1;
Page += 1;
}
else {
break;
}
}
if (DummyTrim != 0) {
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
ASSERT (Mdl->ByteCount != 0);
InPageSupport->ReadOffset.QuadPart += (DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
//
// Shuffle down the PFNs in the MDL.
// Recalculate BasePte to adjust for the shuffle.
//
Pfn1 = MI_PFN_ELEMENT (*Page);
ASSERT (Pfn1->PteAddress->u.Hard.Valid == 0);
ASSERT ((Pfn1->PteAddress->u.Soft.Prototype == 0) &&
(Pfn1->PteAddress->u.Soft.Transition == 1));
InPageSupport->BasePte = Pfn1->PteAddress;
DestinationPage = (PPFN_NUMBER)(Mdl + 1);
do {
*DestinationPage = *Page;
DestinationPage += 1;
Page += 1;
i += 1;
} while (i < MdlPages);
MdlPages -= DummyTrim;
}
//
// Carefully trim trailing dummy pages.
//
ASSERT (MdlPages != 0);
Page = (PPFN_NUMBER)(Mdl + 1) + MdlPages - 1;
if (*Page == DummyPage) {
ASSERT (MdlPages >= 2);
//
// Trim the last page specially as it may be a partial page.
//
Mdl->Size -= sizeof(PFN_NUMBER);
if (BYTE_OFFSET(Mdl->ByteCount) != 0) {
Mdl->ByteCount &= ~(PAGE_SIZE - 1);
}
else {
Mdl->ByteCount -= PAGE_SIZE;
}
ASSERT (Mdl->ByteCount != 0);
DummyPfn1->u3.e2.ReferenceCount -= 1;
//
// Now trim any other trailing pages.
//
Page -= 1;
DummyTrim = 0;
while (Page != ((PPFN_NUMBER)(Mdl + 1))) {
if (*Page != DummyPage) {
break;
}
DummyTrim += 1;
Page -= 1;
}
if (DummyTrim != 0) {
ASSERT (Mdl->Size > (USHORT)(DummyTrim * sizeof(PFN_NUMBER)));
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
}
ASSERT (MdlPages > DummyTrim + 1);
MdlPages -= (DummyTrim + 1);
#if DBG
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
ASSERT (MdlPages == ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa,
Mdl->ByteCount));
#endif
}
//
// If the MDL is not already embedded in the inpage block, see if its
// final size qualifies it - if so, embed it now.
//
if ((Mdl != &InPageSupport->Mdl) &&
(Mdl->ByteCount <= (MM_MAXIMUM_READ_CLUSTER_SIZE + 1) * PAGE_SIZE)){
#if DBG
RtlFillMemoryUlong (&InPageSupport->Page[0],
(MM_MAXIMUM_READ_CLUSTER_SIZE+1) * sizeof (PFN_NUMBER),
0xf1f1f1f1);
#endif
RtlCopyMemory (&InPageSupport->Mdl, Mdl, Mdl->Size);
FreeMdl = Mdl;
Mdl = &InPageSupport->Mdl;
ASSERT (((ULONG_PTR)Mdl & (sizeof(QUAD) - 1)) == 0);
InPageSupport->u1.e1.PrefetchMdlHighBits = ((ULONG_PTR)Mdl >> 3);
}
VOID
MiCheckPfn (
)
/*++
Routine Description:
This routine checks each physical page in the PFN database to ensure
it is in the proper state.
Arguments:
None.
Return Value:
None.
Environment:
Kernel mode, APCs disabled.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER Link, Previous;
ULONG i;
PMMPTE PointerPte;
KIRQL PreviousIrql;
KIRQL OldIrql;
USHORT ValidCheck[4];
USHORT ValidPage[4];
PMMPFN PfnX;
ValidCheck[0] = ValidCheck[1] = ValidCheck[2] = ValidCheck[3] = 0;
ValidPage[0] = ValidPage[1] = ValidPage[2] = ValidPage[3] = 0;
if (CheckPfnBitMap == NULL) {
MiCreateBitMap ( &CheckPfnBitMap, MmNumberOfPhysicalPages, NonPagedPool);
}
RtlClearAllBits (CheckPfnBitMap);
//
// Walk free list.
//
KeRaiseIrql (APC_LEVEL, &PreviousIrql);
LOCK_PFN (OldIrql);
Previous = MM_EMPTY_LIST;
Link = MmFreePageListHead.Flink;
for (i=0; i < MmFreePageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on free list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != FreePageList) {
DbgPrint("page location not freelist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on free list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk zeroed list.
//
Previous = MM_EMPTY_LIST;
Link = MmZeroedPageListHead.Flink;
for (i=0; i < MmZeroedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on zero list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ZeroedPageList) {
DbgPrint("page location not zerolist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on zero list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Bad list.
//
Previous = MM_EMPTY_LIST;
Link = MmBadPageListHead.Flink;
for (i=0; i < MmBadPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Bad list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != BadPageList) {
DbgPrint("page location not Badlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Bad list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Standby list.
//
Previous = MM_EMPTY_LIST;
Link = MmStandbyPageListHead.Flink;
for (i=0; i < MmStandbyPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Standby list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != StandbyPageList) {
DbgPrint("page location not Standbylist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Standby list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in standby list\n");
MiFormatPfn(Pfn1);
} else {
OldIrql = 99;
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame,
&OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Modified list.
//
Previous = MM_EMPTY_LIST;
Link = MmModifiedPageListHead.Flink;
for (i=0; i < MmModifiedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Modified list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ModifiedPageList) {
DbgPrint("page location not Modifiedlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Modified list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in modified list\n");
MiFormatPfn(Pfn1);
} else {
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// All non active pages have been scanned. Locate the
// active pages and make sure they are consistent.
//
//
// set bit zero as page zero is reserved for now
//
RtlSetBits (CheckPfnBitMap, 0L, 1L);
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
while (Link != 0xFFFFFFFF) {
Pfn1 = MI_PFN_ELEMENT (Link);
//
// Make sure the PTE address is okay
//
if ((Pfn1->PteAddress >= (PMMPTE)HYPER_SPACE)
&& (Pfn1->u3.e1.PrototypePte == 0)) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
if (Pfn1->PteAddress < (PMMPTE)PTE_BASE) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
#if defined(_IA64_)
//
// ignore PTEs mapped to IA64 kernel BAT.
//
if (MI_IS_PHYSICAL_ADDRESS(MiGetVirtualAddressMappedByPte(Pfn1->PteAddress))) {
goto NoCheck;
}
#endif // _IA64_
#ifdef _ALPHA_
//
// ignore ptes mapped to ALPHA's 32-bit superpage.
//
if ((Pfn1->PteAddress > (PMMPTE)(ULONG_PTR)0xc0100000) &&
(Pfn1->PteAddress < (PMMPTE)(ULONG_PTR)0xc0180000)) {
goto NoCheck;
}
#endif //ALPHA
//
// Check to make sure the referenced PTE is for this page.
//
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Hard.Valid == 0) {
//
// if the page is a page table page it could be out of
// the working set yet a transition page is keeping it
// around in memory (ups the share count).
//
if ((Pfn1->PteAddress < (PMMPTE)PDE_BASE) ||
(Pfn1->PteAddress > (PMMPTE)PDE_TOP)) {
DbgPrint("Pte not valid for page on active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
}
if (Pfn1->u3.e2.ReferenceCount != 1) {
DbgPrint("refcount not 1\n");
MiFormatPfn(Pfn1);
}
//
// Check to make sure the PTE count for the frame is okay.
//
if (Pfn1->u3.e1.PrototypePte == 1) {
PfnX = MI_PFN_ELEMENT(Pfn1->PteFrame);
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
ValidPage[i] = (USHORT)Pfn1->PteFrame;
}
if (ValidPage[i] == (USHORT)Pfn1->PteFrame) {
ValidCheck[i] += 1;
break;
}
}
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
#if defined(_ALPHA_) || defined(_IA64_)
NoCheck:
#endif
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
}
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
break;
}
PfnX = MI_PFN_ELEMENT(ValidPage[i]);
}
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
LOGICAL
MiZeroAllPageFiles (
VOID
)
/*++
Routine Description:
This routine zeroes all inactive pagefile blocks in all pagefiles.
Arguments:
None.
Return Value:
Returns TRUE on success, FALSE on failure.
Environment:
Kernel mode, the caller must lock down PAGELK.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER MaxPagesToWrite;
KIRQL OldIrql;
ULONG i;
PFN_NUMBER j;
PFN_NUMBER PageFrameIndex;
PMM_ZERO_PAGEFILE_CONTEXT ZeroContext;
ULONG NumberOfPagingFiles;
KEVENT WaitEvents[MAX_PAGE_FILES];
PKEVENT WaitObjects[MAX_PAGE_FILES];
KWAIT_BLOCK WaitBlockArray[MAX_PAGE_FILES];
MaxPagesToWrite = MmModifiedWriteClusterSize;
//
// Get a zeroed page to use as the source for the writes.
//
LOCK_PFN (OldIrql);
MI_DECREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_ALLOCATE_FOR_PAGEFILE_ZEROING);
if (MmAvailablePages < MM_LOW_LIMIT) {
UNLOCK_PFN (OldIrql);
MI_INCREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_FREE_FOR_PAGEFILE_ZEROING);
return TRUE;
}
PageFrameIndex = MiRemoveZeroPage (0);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
Pfn1->u3.e2.ReferenceCount = (USHORT) MaxPagesToWrite;
Pfn1->PteAddress = (PMMPTE) (ULONG_PTR)(X64K | 0x1);
Pfn1->OriginalPte.u.Long = 0;
MI_SET_PFN_DELETED (Pfn1);
UNLOCK_PFN (OldIrql);
//
// Capture the number of paging files in case a new one gets added.
//
NumberOfPagingFiles = MmNumberOfPagingFiles;
for (i = NumberOfPagingFiles; i != 0; i -= 1) {
KeInitializeEvent (&WaitEvents[i - 1], NotificationEvent, FALSE);
WaitObjects[i - 1] = &WaitEvents[i - 1];
ZeroContext = ExAllocatePoolWithTag (NonPagedPool,
sizeof (MM_ZERO_PAGEFILE_CONTEXT),
'wZmM');
if (ZeroContext == NULL) {
KeSetEvent (WaitObjects[i - 1], 0, FALSE);
continue;
}
ZeroContext->PagingFile = MmPagingFile[i - 1];
ZeroContext->ZeroedPageFrame = PageFrameIndex;
ZeroContext->AllDone = WaitObjects[i - 1];
if (i != 1) {
ExInitializeWorkItem (&ZeroContext->WorkItem,
MiZeroPageFile,
(PVOID) ZeroContext);
ExQueueWorkItem (&ZeroContext->WorkItem, CriticalWorkQueue);
}
else {
//
// Zero the first pagefile ourself, then wait for
// any others to finish.
//
KeSetEvent (WaitObjects[i - 1], 0, FALSE);
MiZeroPageFile (ZeroContext);
}
}
if (NumberOfPagingFiles > 1) {
KeWaitForMultipleObjects (NumberOfPagingFiles,
&WaitObjects[0],
WaitAll,
Executive,
KernelMode,
FALSE,
NULL,
&WaitBlockArray[0]);
}
LOCK_PFN (OldIrql);
ASSERT (Pfn1->u3.e2.ReferenceCount >= MaxPagesToWrite);
if (Pfn1->u3.e2.ReferenceCount == MaxPagesToWrite) {
MI_INCREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_FREE_FOR_PAGEFILE_ZEROING);
}
for (j = 0; j < MaxPagesToWrite; j += 1) {
MiDecrementReferenceCountInline (Pfn1, PageFrameIndex);
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
