VOID
MiFlushUnusedSections (
VOID
)
/*++
Routine Description:
This routine rumages through the PFN database and attempts
to close any unused sections.
Arguments:
None.
Return Value:
None.
--*/
{
PMMPFN LastPfn;
PMMPFN Pfn1;
PSUBSECTION Subsection;
KIRQL OldIrql;
LOCK_PFN (OldIrql);
Pfn1 = MI_PFN_ELEMENT (MmLowestPhysicalPage + 1);
LastPfn = MI_PFN_ELEMENT(MmHighestPhysicalPage);
while (Pfn1 < LastPfn) {
if (Pfn1->OriginalPte.u.Soft.Prototype == 1) {
if ((Pfn1->u3.e1.PageLocation == ModifiedPageList) ||
(Pfn1->u3.e1.PageLocation == StandbyPageList)) {
//
// Make sure the PTE is not waiting for I/O to complete.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
Subsection = MiGetSubsectionAddress (&Pfn1->OriginalPte);
MiFlushUnusedSectionInternal (Subsection->ControlArea);
}
}
}
Pfn1++;
}
UNLOCK_PFN (OldIrql);
return;
}
VOID
MiDumpPfn ( )
{
ULONG i;
PMMPFN Pfn1;
Pfn1 = MI_PFN_ELEMENT (MmLowestPhysicalPage);
for (i=0; i < MmNumberOfPhysicalPages; i++) {
MiFormatPfn (Pfn1);
Pfn1++;
}
return;
}
NTSTATUS
MmAddPhysicalMemory (
IN PPHYSICAL_ADDRESS StartAddress,
IN OUT PLARGE_INTEGER NumberOfBytes
)
/*++
Routine Description:
This routine adds the specified physical address range to the system.
This includes initializing PFN database entries and adding it to the
freelists.
Arguments:
StartAddress - Supplies the starting physical address.
NumberOfBytes - Supplies a pointer to the number of bytes being added.
If any bytes were added (ie: STATUS_SUCCESS is being
returned), the actual amount is returned here.
Return Value:
NTSTATUS.
Environment:
Kernel mode. PASSIVE level. No locks held.
--*/
{
ULONG i;
PMMPFN Pfn1;
KIRQL OldIrql;
LOGICAL Inserted;
LOGICAL Updated;
MMPTE TempPte;
PMMPTE PointerPte;
PMMPTE LastPte;
PFN_NUMBER NumberOfPages;
PFN_NUMBER start;
PFN_NUMBER count;
PFN_NUMBER StartPage;
PFN_NUMBER EndPage;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER Page;
PFN_NUMBER LastPage;
PFN_COUNT PagesNeeded;
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
PPHYSICAL_MEMORY_RUN NewRun;
LOGICAL PfnDatabaseIsPhysical;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
//
// The system must be configured for dynamic memory addition. This is
// critical as only then is the database guaranteed to be non-sparse.
//
if (MmDynamicPfn == FALSE) {
return STATUS_NOT_SUPPORTED;
}
PfnDatabaseIsPhysical = TRUE;
}
else {
PfnDatabaseIsPhysical = FALSE;
}
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
EndPage = StartPage + NumberOfPages;
if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
//
// Truncate the request into something that can be mapped by the PFN
// database.
//
EndPage = MmHighestPossiblePhysicalPage + 1;
NumberOfPages = EndPage - StartPage;
}
//
// The range cannot wrap.
//
if (StartPage >= EndPage) {
return STATUS_INVALID_PARAMETER_1;
}
ExAcquireFastMutex (&MmDynamicMemoryMutex);
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
i,
' mM');
if (NewPhysicalMemoryBlock == NULL) {
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// The range cannot overlap any ranges that are already present.
//
start = 0;
LOCK_PFN (OldIrql);
do {
count = MmPhysicalMemoryBlock->Run[start].PageCount;
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
if (count != 0) {
LastPage = Page + count;
if ((StartPage < Page) && (EndPage > Page)) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_CONFLICTING_ADDRESSES;
}
if ((StartPage >= Page) && (StartPage < LastPage)) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_CONFLICTING_ADDRESSES;
}
}
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Fill any gaps in the (sparse) PFN database needed for these pages,
// unless the PFN database was physically allocated and completely
// committed up front.
//
PagesNeeded = 0;
if (PfnDatabaseIsPhysical == FALSE) {
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
while (PointerPte <= LastPte) {
if (PointerPte->u.Hard.Valid == 0) {
PagesNeeded += 1;
}
PointerPte += 1;
}
if (MmAvailablePages < PagesNeeded) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_INSUFFICIENT_RESOURCES;
}
TempPte = ValidKernelPte;
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
while (PointerPte <= LastPte) {
if (PointerPte->u.Hard.Valid == 0) {
PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
MiInitializePfn (PageFrameIndex, PointerPte, 0);
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
*PointerPte = TempPte;
}
PointerPte += 1;
}
MmResidentAvailablePages -= PagesNeeded;
}
//
// If the new range is adjacent to an existing range, just merge it into
// the old block. Otherwise use the new block as a new entry will have to
// be used.
//
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;
NewRun = &NewPhysicalMemoryBlock->Run[0];
start = 0;
Inserted = FALSE;
Updated = FALSE;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
count = MmPhysicalMemoryBlock->Run[start].PageCount;
if (Inserted == FALSE) {
//
// Note overlaps into adjacent ranges were already checked above.
//
if (StartPage == Page + count) {
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
//
// Coalesce below and above to avoid leaving zero length gaps
// as these gaps would prevent callers from removing ranges
// the span them.
//
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
start += 1;
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
count = MmPhysicalMemoryBlock->Run[start].PageCount;
if (StartPage + NumberOfPages == Page) {
MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
count;
MmPhysicalMemoryBlock->NumberOfRuns -= 1;
//
// Copy any remaining entries.
//
if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
&MmPhysicalMemoryBlock->Run[start + 1],
(MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
}
}
}
Updated = TRUE;
break;
}
if (StartPage + NumberOfPages == Page) {
MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
Updated = TRUE;
break;
}
if (StartPage + NumberOfPages <= Page) {
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
//
// Don't insert here - the new entry really belongs
// (at least) one entry further down.
//
continue;
}
}
NewRun->BasePage = StartPage;
NewRun->PageCount = NumberOfPages;
NewRun += 1;
Inserted = TRUE;
Updated = TRUE;
}
}
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// If the memory block has not been updated, then the new entry must
// be added at the very end.
//
if (Updated == FALSE) {
ASSERT (Inserted == FALSE);
NewRun->BasePage = StartPage;
NewRun->PageCount = NumberOfPages;
Inserted = TRUE;
}
//
// Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
// releasing the PFN lock.
//
if (Inserted == TRUE) {
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
}
//
// Note that the page directory (page parent entries on Win64) must be
// filled in at system boot so that already-created processes do not fault
// when referencing the new PFNs.
//
//
// Walk through the memory descriptors and add pages to the
// free list in the PFN database.
//
PageFrameIndex = StartPage;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
if (EndPage - 1 > MmHighestPhysicalPage) {
MmHighestPhysicalPage = EndPage - 1;
}
while (PageFrameIndex < EndPage) {
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ShortFlags == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
ASSERT (Pfn1->PteFrame == 0);
ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
(Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));
//
// Set the PTE address to the physical page for
// virtual address alignment checking.
//
Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);
MiInsertPageInList (MmPageLocationList[FreePageList],
PageFrameIndex);
PageFrameIndex += 1;
Pfn1 += 1;
}
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;
UNLOCK_PFN (OldIrql);
//
// Increase all commit limits to reflect the additional memory.
//
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
MmTotalCommitLimit += NumberOfPages;
MmTotalCommitLimitMaximum += NumberOfPages;
MmTotalCommittedPages += PagesNeeded;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (OldPhysicalMemoryBlock);
//
// Indicate number of bytes actually added to our caller.
//
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
return STATUS_SUCCESS;
}
PFN_COUNT
MiRemovePhysicalPages (
IN PFN_NUMBER StartPage,
IN PFN_NUMBER EndPage
)
/*++
Routine Description:
This routine searches the PFN database for free, zeroed or standby pages
that are marked for removal.
Arguments:
StartPage - Supplies the low physical frame number to remove.
EndPage - Supplies the last physical frame number to remove.
Return Value:
Returns the number of pages removed from the free, zeroed and standby lists.
Environment:
Kernel mode, PFN lock held.
--*/
{
PMMPFN Pfn1;
PMMPFN Pfn2;
PMMPFN PfnNextColored;
PMMPFN PfnNextFlink;
PMMPFN PfnLastColored;
PFN_NUMBER Page;
LOGICAL RemovePage;
ULONG Color;
PMMCOLOR_TABLES ColorHead;
PFN_NUMBER MovedPage;
MMLISTS MemoryList;
PFN_NUMBER PageNextColored;
PFN_NUMBER PageNextFlink;
PFN_NUMBER PageLastColored;
PFN_COUNT NumberOfPages;
PMMPFNLIST ListHead;
LOGICAL RescanNeeded;
MM_PFN_LOCK_ASSERT();
NumberOfPages = 0;
rescan:
//
// Grab all zeroed (and then free) pages first directly from the
// colored lists to avoid multiple walks down these singly linked lists.
// Handle transition pages last.
//
for (MemoryList = ZeroedPageList; MemoryList <= FreePageList; MemoryList += 1) {
ListHead = MmPageLocationList[MemoryList];
for (Color = 0; Color < MmSecondaryColors; Color += 1) {
ColorHead = &MmFreePagesByColor[MemoryList][Color];
MovedPage = MM_EMPTY_LIST;
while (ColorHead->Flink != MM_EMPTY_LIST) {
Page = ColorHead->Flink;
Pfn1 = MI_PFN_ELEMENT(Page);
ASSERT ((MMLISTS)Pfn1->u3.e1.PageLocation == MemoryList);
//
// The Flink and Blink must be nonzero here for the page
// to be on the listhead. Only code that scans the
// MmPhysicalMemoryBlock has to check for the zero case.
//
ASSERT (Pfn1->u1.Flink != 0);
ASSERT (Pfn1->u2.Blink != 0);
//
// See if the page is desired by the caller.
//
if (Pfn1->u3.e1.RemovalRequested == 1) {
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
MiUnlinkFreeOrZeroedPage (Page);
MiInsertPageInList (MmPageLocationList[BadPageList],
Page);
NumberOfPages += 1;
}
else {
//
// Unwanted so put the page on the end of list.
// If first time, save pfn.
//
if (MovedPage == MM_EMPTY_LIST) {
MovedPage = Page;
}
else if (Page == MovedPage) {
//
// No more pages available in this colored chain.
//
break;
}
//
// If the colored chain has more than one entry then
// put this page on the end.
//
PageNextColored = (PFN_NUMBER)Pfn1->OriginalPte.u.Long;
if (PageNextColored == MM_EMPTY_LIST) {
//
// No more pages available in this colored chain.
//
break;
}
ASSERT (Pfn1->u1.Flink != 0);
ASSERT (Pfn1->u1.Flink != MM_EMPTY_LIST);
ASSERT (Pfn1->PteFrame != MI_MAGIC_AWE_PTEFRAME);
PfnNextColored = MI_PFN_ELEMENT(PageNextColored);
ASSERT ((MMLISTS)PfnNextColored->u3.e1.PageLocation == MemoryList);
ASSERT (PfnNextColored->PteFrame != MI_MAGIC_AWE_PTEFRAME);
//
// Adjust the free page list so Page
// follows PageNextFlink.
//
PageNextFlink = Pfn1->u1.Flink;
PfnNextFlink = MI_PFN_ELEMENT(PageNextFlink);
ASSERT ((MMLISTS)PfnNextFlink->u3.e1.PageLocation == MemoryList);
ASSERT (PfnNextFlink->PteFrame != MI_MAGIC_AWE_PTEFRAME);
PfnLastColored = ColorHead->Blink;
ASSERT (PfnLastColored != (PMMPFN)MM_EMPTY_LIST);
ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
ASSERT (PfnLastColored->PteFrame != MI_MAGIC_AWE_PTEFRAME);
ASSERT (PfnLastColored->u2.Blink != MM_EMPTY_LIST);
ASSERT ((MMLISTS)PfnLastColored->u3.e1.PageLocation == MemoryList);
PageLastColored = PfnLastColored - MmPfnDatabase;
if (ListHead->Flink == Page) {
ASSERT (Pfn1->u2.Blink == MM_EMPTY_LIST);
ASSERT (ListHead->Blink != Page);
ListHead->Flink = PageNextFlink;
PfnNextFlink->u2.Blink = MM_EMPTY_LIST;
}
else {
ASSERT (Pfn1->u2.Blink != MM_EMPTY_LIST);
ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->PteFrame != MI_MAGIC_AWE_PTEFRAME);
ASSERT ((MMLISTS)(MI_PFN_ELEMENT((MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink)))->u3.e1.PageLocation == MemoryList);
MI_PFN_ELEMENT(Pfn1->u2.Blink)->u1.Flink = PageNextFlink;
PfnNextFlink->u2.Blink = Pfn1->u2.Blink;
}
#if DBG
if (PfnLastColored->u1.Flink == MM_EMPTY_LIST) {
ASSERT (ListHead->Blink == PageLastColored);
}
#endif
Pfn1->u1.Flink = PfnLastColored->u1.Flink;
Pfn1->u2.Blink = PageLastColored;
if (ListHead->Blink == PageLastColored) {
ListHead->Blink = Page;
}
//
// Adjust the colored chains.
//
if (PfnLastColored->u1.Flink != MM_EMPTY_LIST) {
ASSERT (MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->PteFrame != MI_MAGIC_AWE_PTEFRAME);
ASSERT ((MMLISTS)(MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u3.e1.PageLocation) == MemoryList);
MI_PFN_ELEMENT(PfnLastColored->u1.Flink)->u2.Blink = Page;
}
PfnLastColored->u1.Flink = Page;
ColorHead->Flink = PageNextColored;
Pfn1->OriginalPte.u.Long = MM_EMPTY_LIST;
ASSERT (PfnLastColored->OriginalPte.u.Long == MM_EMPTY_LIST);
PfnLastColored->OriginalPte.u.Long = Page;
ColorHead->Blink = Pfn1;
}
}
}
}
RescanNeeded = FALSE;
Pfn1 = MI_PFN_ELEMENT (StartPage);
do {
if ((Pfn1->u3.e1.PageLocation == StandbyPageList) &&
(Pfn1->u1.Flink != 0) &&
(Pfn1->u2.Blink != 0) &&
(Pfn1->u3.e2.ReferenceCount == 0)) {
ASSERT (Pfn1->u3.e1.ReadInProgress == 0);
RemovePage = TRUE;
if (Pfn1->u3.e1.RemovalRequested == 0) {
//
// This page is not directly needed for a hot remove - but if
// it contains a chunk of prototype PTEs (and this chunk is
// in a page that needs to be removed), then any pages
// referenced by transition prototype PTEs must also be removed
// before the desired page can be removed.
//
// The same analogy holds for page parent, directory and table
// pages.
//
Pfn2 = MI_PFN_ELEMENT (Pfn1->PteFrame);
if (Pfn2->u3.e1.RemovalRequested == 0) {
#if defined (_WIN64)
Pfn2 = MI_PFN_ELEMENT (Pfn2->PteFrame);
if (Pfn2->u3.e1.RemovalRequested == 0) {
RemovePage = FALSE;
}
else if (Pfn2->u2.ShareCount == 1) {
RescanNeeded = TRUE;
}
#else
RemovePage = FALSE;
#endif
}
else if (Pfn2->u2.ShareCount == 1) {
RescanNeeded = TRUE;
}
}
if (RemovePage == TRUE) {
//
// This page is in the desired range - grab it.
//
MiUnlinkPageFromList (Pfn1);
MiRestoreTransitionPte (StartPage);
MiInsertPageInList (MmPageLocationList[BadPageList],
StartPage);
NumberOfPages += 1;
}
}
StartPage += 1;
Pfn1 += 1;
} while (StartPage < EndPage);
if (RescanNeeded == TRUE) {
//
// A page table, directory or parent was freed by removing a transition
// page from the cache. Rescan from the top to pick it up.
//
#if DBG
MiDynmemData[7] += 1;
#endif
goto rescan;
}
#if DBG
else {
MiDynmemData[8] += 1;
}
#endif
return NumberOfPages;
}
VOID
MiDeletePte (
IN PMMPTE PointerPte,
IN PVOID VirtualAddress,
IN ULONG AddressSpaceDeletion,
IN PEPROCESS CurrentProcess,
IN PMMPTE PrototypePte,
IN PMMPTE_FLUSH_LIST PteFlushList OPTIONAL
)
/*++
Routine Description:
This routine deletes the contents of the specified PTE. The PTE
can be in one of the following states:
- active and valid
- transition
- in paging file
- in prototype PTE format
Arguments:
PointerPte - Supplies a pointer to the PTE to delete.
VirtualAddress - Supplies the virtual address which corresponds to
the PTE. This is used to locate the working set entry
to eliminate it.
AddressSpaceDeletion - Supplies TRUE if the address space is being
deleted, FALSE otherwise. If TRUE is specified
the TB is not flushed and valid addresses are
not removed from the working set.
CurrentProcess - Supplies a pointer to the current process.
PrototypePte - Supplies a pointer to the prototype PTE which currently
or originally mapped this page. This is used to determine
if pte is a fork PTE and should have it's reference block
decremented.
Return Value:
None.
Environment:
Kernel mode, APCs disabled, PFN lock and working set mutex held.
--*/
{
PMMPTE PointerPde;
PMMPFN Pfn1;
PMMPFN Pfn2;
MMPTE PteContents;
ULONG WorkingSetIndex;
ULONG Entry;
PVOID SwapVa;
MMWSLENTRY Locked;
ULONG WsPfnIndex;
PMMCLONE_BLOCK CloneBlock;
PMMCLONE_DESCRIPTOR CloneDescriptor;
MM_PFN_LOCK_ASSERT();
#if DBG
if (MmDebug & MM_DBG_PTE_UPDATE) {
DbgPrint("deleting PTE\n");
MiFormatPte(PointerPte);
}
#endif //DBG
PteContents = *PointerPte;
if (PteContents.u.Hard.Valid == 1) {
#ifdef R4000
ASSERT (PteContents.u.Hard.Global == 0);
#endif
#ifdef _X86_
#if DBG
#if !defined(NT_UP)
if (PteContents.u.Hard.Writable == 1) {
ASSERT (PteContents.u.Hard.Dirty == 1);
}
ASSERT (PteContents.u.Hard.Accessed == 1);
#endif //NTUP
#endif //DBG
#endif //X86
//
// Pte is valid. Check PFN database to see if this is a prototype PTE.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
WsPfnIndex = Pfn1->u1.WsIndex;
#if DBG
if (MmDebug & MM_DBG_PTE_UPDATE) {
MiFormatPfn(Pfn1);
}
#endif //DBG
CloneDescriptor = NULL;
if (Pfn1->u3.e1.PrototypePte == 1) {
CloneBlock = (PMMCLONE_BLOCK)Pfn1->PteAddress;
//
// Capture the state of the modified bit for this
// pte.
//
MI_CAPTURE_DIRTY_BIT_TO_PFN (PointerPte, Pfn1);
//
// Decrement the share and valid counts of the page table
// page which maps this PTE.
//
PointerPde = MiGetPteAddress (PointerPte);
MiDecrementShareAndValidCount (PointerPde->u.Hard.PageFrameNumber);
//
// Decrement the share count for the physical page.
//
MiDecrementShareCount (PteContents.u.Hard.PageFrameNumber);
//
// Check to see if this is a fork prototype PTE and if so
// update the clone descriptor address.
//
if (PointerPte <= MiGetPteAddress(MM_HIGHEST_USER_ADDRESS)) {
if (PrototypePte != Pfn1->PteAddress) {
//
// Locate the clone descriptor within the clone tree.
//
CloneDescriptor = MiLocateCloneAddress ((PVOID)CloneBlock);
#if DBG
if (CloneDescriptor == NULL) {
DbgPrint("1PrototypePte %lx Clone desc %lx pfn pte addr %lx\n",
PrototypePte, CloneDescriptor, Pfn1->PteAddress);
MiFormatPte(PointerPte);
ASSERT (FALSE);
}
#endif // DBG
}
}
} else {
//
// This pte is a NOT a prototype PTE, delete the physical page.
//
//
// Decrement the share and valid counts of the page table
// page which maps this PTE.
//
MiDecrementShareAndValidCount (Pfn1->PteFrame);
MI_SET_PFN_DELETED (Pfn1);
//
// Decrement the share count for the physical page. As the page
// is private it will be put on the free list.
//
MiDecrementShareCountOnly (PteContents.u.Hard.PageFrameNumber);
//
// Decrement the count for the number of private pages.
//
CurrentProcess->NumberOfPrivatePages -= 1;
}
//
// Find the WSLE for this page and eliminate it.
//
//
// If we are deleting the system portion of the address space, do
// not remove WSLEs or flush translation buffers as there can be
// no other usage of this address space.
//
if (AddressSpaceDeletion == FALSE) {
WorkingSetIndex = MiLocateWsle (VirtualAddress,
MmWorkingSetList,
WsPfnIndex );
ASSERT (WorkingSetIndex != WSLE_NULL_INDEX);
//
// Check to see if this entry is locked in the working set
// or locked in memory.
//
Locked = MmWsle[WorkingSetIndex].u1.e1;
MiRemoveWsle (WorkingSetIndex, MmWorkingSetList);
//
// Add this entry to the list of free working set entries
// and adjust the working set count.
//
MiReleaseWsle (WorkingSetIndex, &CurrentProcess->Vm);
if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {
//
// This entry is locked.
//
ASSERT (WorkingSetIndex < MmWorkingSetList->FirstDynamic);
MmWorkingSetList->FirstDynamic -= 1;
if (WorkingSetIndex != MmWorkingSetList->FirstDynamic) {
Entry = MmWorkingSetList->FirstDynamic;
ASSERT (MmWsle[Entry].u1.e1.Valid);
SwapVa = MmWsle[Entry].u1.VirtualAddress;
SwapVa = PAGE_ALIGN (SwapVa);
Pfn2 = MI_PFN_ELEMENT (
MiGetPteAddress (SwapVa)->u.Hard.PageFrameNumber);
#if 0
Entry = MiLocateWsleAndParent (SwapVa,
&Parent,
MmWorkingSetList,
Pfn2->u1.WsIndex);
//
// Swap the removed entry with the last locked entry
// which is located at first dynamic.
//
MiSwapWslEntries (Entry,
Parent,
WorkingSetIndex,
MmWorkingSetList);
#endif //0
MiSwapWslEntries (Entry,
WorkingSetIndex,
&CurrentProcess->Vm);
}
} else {
ASSERT (WorkingSetIndex >= MmWorkingSetList->FirstDynamic);
}
//
// Flush the entry out of the TB.
//
if (!ARGUMENT_PRESENT (PteFlushList)) {
KeFlushSingleTb (VirtualAddress,
TRUE,
FALSE,
(PHARDWARE_PTE)PointerPte,
ZeroPte.u.Flush);
} else {
if (PteFlushList->Count != MM_MAXIMUM_FLUSH_COUNT) {
PteFlushList->FlushPte[PteFlushList->Count] = PointerPte;
PteFlushList->FlushVa[PteFlushList->Count] = VirtualAddress;
PteFlushList->Count += 1;
}
*PointerPte = ZeroPte;
}
if (CloneDescriptor != NULL) {
//
// Flush PTEs as this could release the PFN_LOCK.
//
MiFlushPteList (PteFlushList, FALSE, ZeroPte);
//
// Decrement the reference count for the clone block,
// note that this could release and reacquire
// the mutexes hence cannot be done until after the
// working set index has been removed.
//
if (MiDecrementCloneBlockReference ( CloneDescriptor,
CloneBlock,
CurrentProcess )) {
//
// The working set mutex was released. This may
// have removed the current page table page.
//
MiDoesPdeExistAndMakeValid (PointerPde,
CurrentProcess,
TRUE);
}
}
}
} else if (PteContents.u.Soft.Prototype == 1) {
//
// This is a prototype PTE, if it is a fork PTE clean up the
// fork structures.
//
if (PteContents.u.Soft.PageFileHigh != 0xFFFFF) {
//
// Check to see if the prototype PTE is a fork prototype PTE.
//
if (PointerPte <= MiGetPteAddress(MM_HIGHEST_USER_ADDRESS)) {
if (PrototypePte != MiPteToProto (PointerPte)) {
CloneBlock = (PMMCLONE_BLOCK)MiPteToProto (PointerPte);
CloneDescriptor = MiLocateCloneAddress ((PVOID)CloneBlock);
#if DBG
if (CloneDescriptor == NULL) {
DbgPrint("1PrototypePte %lx Clone desc %lx \n",
PrototypePte, CloneDescriptor);
MiFormatPte(PointerPte);
ASSERT (FALSE);
}
#endif //DBG
//
// Decrement the reference count for the clone block,
// note that this could release and reacquire
// the mutexes.
//
*PointerPte = ZeroPte;
MiFlushPteList (PteFlushList, FALSE, ZeroPte);
if (MiDecrementCloneBlockReference ( CloneDescriptor,
CloneBlock,
CurrentProcess )) {
//
// The working set mutex was released. This may
// have removed the current page table page.
//
MiDoesPdeExistAndMakeValid (MiGetPteAddress (PointerPte),
CurrentProcess,
TRUE);
}
}
}
}
} else if (PteContents.u.Soft.Transition == 1) {
//
// This is a transition PTE. (Page is private)
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
MI_SET_PFN_DELETED (Pfn1);
MiDecrementShareCount (Pfn1->PteFrame);
//
// Check the reference count for the page, if the reference
// count is zero, move the page to the free list, if the reference
// count is not zero, ignore this page. When the refernce count
// goes to zero, it will be placed on the free list.
//
if (Pfn1->u3.e2.ReferenceCount == 0) {
MiUnlinkPageFromList (Pfn1);
MiReleasePageFileSpace (Pfn1->OriginalPte);
MiInsertPageInList (MmPageLocationList[FreePageList],
PteContents.u.Trans.PageFrameNumber);
}
//
// Decrement the count for the number of private pages.
//
CurrentProcess->NumberOfPrivatePages -= 1;
} else {
//
// Must be page file space.
//
if (PteContents.u.Soft.PageFileHigh != 0) {
if (MiReleasePageFileSpace (*PointerPte)) {
//
// Decrement the count for the number of private pages.
//
CurrentProcess->NumberOfPrivatePages -= 1;
}
}
}
//
// Zero the PTE contents.
//
*PointerPte = ZeroPte;
return;
}
LOGICAL
MiZeroAllPageFiles (
VOID
)
/*++
Routine Description:
This routine zeroes all inactive pagefile blocks in all pagefiles.
Arguments:
None.
Return Value:
Returns TRUE on success, FALSE on failure.
Environment:
Kernel mode, the caller must lock down PAGELK.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER MaxPagesToWrite;
KIRQL OldIrql;
ULONG i;
PFN_NUMBER j;
PFN_NUMBER PageFrameIndex;
PMM_ZERO_PAGEFILE_CONTEXT ZeroContext;
ULONG NumberOfPagingFiles;
KEVENT WaitEvents[MAX_PAGE_FILES];
PKEVENT WaitObjects[MAX_PAGE_FILES];
KWAIT_BLOCK WaitBlockArray[MAX_PAGE_FILES];
MaxPagesToWrite = MmModifiedWriteClusterSize;
//
// Get a zeroed page to use as the source for the writes.
//
LOCK_PFN (OldIrql);
MI_DECREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_ALLOCATE_FOR_PAGEFILE_ZEROING);
if (MmAvailablePages < MM_LOW_LIMIT) {
UNLOCK_PFN (OldIrql);
MI_INCREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_FREE_FOR_PAGEFILE_ZEROING);
return TRUE;
}
PageFrameIndex = MiRemoveZeroPage (0);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
Pfn1->u3.e2.ReferenceCount = (USHORT) MaxPagesToWrite;
Pfn1->PteAddress = (PMMPTE) (ULONG_PTR)(X64K | 0x1);
Pfn1->OriginalPte.u.Long = 0;
MI_SET_PFN_DELETED (Pfn1);
UNLOCK_PFN (OldIrql);
//
// Capture the number of paging files in case a new one gets added.
//
NumberOfPagingFiles = MmNumberOfPagingFiles;
for (i = NumberOfPagingFiles; i != 0; i -= 1) {
KeInitializeEvent (&WaitEvents[i - 1], NotificationEvent, FALSE);
WaitObjects[i - 1] = &WaitEvents[i - 1];
ZeroContext = ExAllocatePoolWithTag (NonPagedPool,
sizeof (MM_ZERO_PAGEFILE_CONTEXT),
'wZmM');
if (ZeroContext == NULL) {
KeSetEvent (WaitObjects[i - 1], 0, FALSE);
continue;
}
ZeroContext->PagingFile = MmPagingFile[i - 1];
ZeroContext->ZeroedPageFrame = PageFrameIndex;
ZeroContext->AllDone = WaitObjects[i - 1];
if (i != 1) {
ExInitializeWorkItem (&ZeroContext->WorkItem,
MiZeroPageFile,
(PVOID) ZeroContext);
ExQueueWorkItem (&ZeroContext->WorkItem, CriticalWorkQueue);
}
else {
//
// Zero the first pagefile ourself, then wait for
// any others to finish.
//
KeSetEvent (WaitObjects[i - 1], 0, FALSE);
MiZeroPageFile (ZeroContext);
}
}
if (NumberOfPagingFiles > 1) {
KeWaitForMultipleObjects (NumberOfPagingFiles,
&WaitObjects[0],
WaitAll,
Executive,
KernelMode,
FALSE,
NULL,
&WaitBlockArray[0]);
}
LOCK_PFN (OldIrql);
ASSERT (Pfn1->u3.e2.ReferenceCount >= MaxPagesToWrite);
if (Pfn1->u3.e2.ReferenceCount == MaxPagesToWrite) {
MI_INCREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_FREE_FOR_PAGEFILE_ZEROING);
}
for (j = 0; j < MaxPagesToWrite; j += 1) {
MiDecrementReferenceCountInline (Pfn1, PageFrameIndex);
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
NTSTATUS
MiCcPutPagesInTransition (
IN PMI_READ_INFO MiReadInfo
)
/*++
Routine Description:
This routine allocates physical memory for the specified read-list and
puts all the pages in transition (so collided faults from other threads
for these same pages remain coherent). I/O for any pages not already
resident are issued here. The caller must wait for their completion.
Arguments:
MiReadInfo - Supplies a pointer to the read-list.
Return Value:
STATUS_SUCCESS - all the pages were already resident, reference counts
have been applied and no I/O needs to be waited for.
STATUS_ISSUE_PAGING_IO - the I/O has been issued and the caller must wait.
Various other failure status values indicate the operation failed.
Environment:
Kernel mode. PASSIVE_LEVEL.
--*/
{
NTSTATUS status;
PMMPTE LocalPrototypePte;
PVOID StartingVa;
PFN_NUMBER MdlPages;
KIRQL OldIrql;
MMPTE PteContents;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER ResidentAvailableCharge;
PPFN_NUMBER IoPage;
PPFN_NUMBER ApiPage;
PPFN_NUMBER Page;
PPFN_NUMBER DestinationPage;
ULONG PageColor;
PMMPTE PointerPte;
PMMPTE *ProtoPteArray;
PMMPTE *EndProtoPteArray;
PFN_NUMBER DummyPage;
PMDL Mdl;
PMDL FreeMdl;
PMMPFN PfnProto;
PMMPFN Pfn1;
PMMPFN DummyPfn1;
ULONG i;
PFN_NUMBER DummyTrim;
ULONG NumberOfPagesNeedingIo;
MMPTE TempPte;
PMMPTE PointerPde;
PEPROCESS CurrentProcess;
PMMINPAGE_SUPPORT InPageSupport;
PKPRCB Prcb;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
MiReadInfo->DummyPagePfn = NULL;
FreeMdl = NULL;
CurrentProcess = PsGetCurrentProcess();
PfnProto = NULL;
PointerPde = NULL;
InPageSupport = MiReadInfo->InPageSupport;
Mdl = MI_EXTRACT_PREFETCH_MDL (InPageSupport);
ASSERT (Mdl == MiReadInfo->IoMdl);
IoPage = (PPFN_NUMBER)(Mdl + 1);
ApiPage = (PPFN_NUMBER)(MiReadInfo->ApiMdl + 1);
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
MdlPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES (StartingVa,
Mdl->ByteCount);
if (MdlPages + 1 > MAXUSHORT) {
//
// The PFN ReferenceCount for the dummy page could wrap, refuse the
// request.
//
return STATUS_INSUFFICIENT_RESOURCES;
}
NumberOfPagesNeedingIo = 0;
ProtoPteArray = (PMMPTE *)InPageSupport->BasePte;
EndProtoPteArray = ProtoPteArray + MdlPages;
ASSERT (*ProtoPteArray != NULL);
LOCK_PFN (OldIrql);
//
// Ensure sufficient pages exist for the transfer plus the dummy page.
//
if (((SPFN_NUMBER)MdlPages > (SPFN_NUMBER)(MmAvailablePages - MM_HIGH_LIMIT)) ||
(MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MdlPages)) {
UNLOCK_PFN (OldIrql);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// Charge resident available immediately as the PFN lock may get released
// and reacquired below before all the pages have been locked down.
// Note the dummy page is immediately charged separately.
//
MI_DECREMENT_RESIDENT_AVAILABLE (MdlPages, MM_RESAVAIL_ALLOCATE_BUILDMDL);
ResidentAvailableCharge = MdlPages;
//
// Allocate a dummy page to map discarded pages that aren't skipped.
//
DummyPage = MiRemoveAnyPage (0);
Pfn1 = MI_PFN_ELEMENT (DummyPage);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MiInitializePfnForOtherProcess (DummyPage, MI_PF_DUMMY_PAGE_PTE, 0);
//
// Give the page a containing frame so MiIdentifyPfn won't crash.
//
Pfn1->u4.PteFrame = PsInitialSystemProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT;
//
// Always bias the reference count by 1 and charge for this locked page
// up front so the myriad increments and decrements don't get slowed
// down with needless checking.
//
Pfn1->u3.e1.PrototypePte = 0;
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
Pfn1->u3.e1.ReadInProgress = 1;
MiReadInfo->DummyPagePfn = Pfn1;
DummyPfn1 = Pfn1;
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount + MdlPages);
//
// Properly initialize the inpage support block fields we overloaded.
//
InPageSupport->BasePte = *ProtoPteArray;
//
// Build the proper InPageSupport and MDL to describe this run.
//
for (; ProtoPteArray < EndProtoPteArray; ProtoPteArray += 1, IoPage += 1, ApiPage += 1) {
//
// Fill the MDL entry for this RLE.
//
PointerPte = *ProtoPteArray;
ASSERT (PointerPte != NULL);
//
// The PointerPte better be inside a prototype PTE allocation
// so that subsequent page trims update the correct PTEs.
//
ASSERT (((PointerPte >= (PMMPTE)MmPagedPoolStart) &&
(PointerPte <= (PMMPTE)MmPagedPoolEnd)) ||
((PointerPte >= (PMMPTE)MmSpecialPoolStart) && (PointerPte <= (PMMPTE)MmSpecialPoolEnd)));
//
// Check the state of this prototype PTE now that the PFN lock is held.
// If the page is not resident, the PTE must be put in transition with
// read in progress before the PFN lock is released.
//
//
// Lock page containing prototype PTEs in memory by
// incrementing the reference count for the page.
// Unlock any page locked earlier containing prototype PTEs if
// the containing page is not the same for both.
//
if (PfnProto != NULL) {
if (PointerPde != MiGetPteAddress (PointerPte)) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
PfnProto = NULL;
}
}
if (PfnProto == NULL) {
ASSERT (!MI_IS_PHYSICAL_ADDRESS (PointerPte));
PointerPde = MiGetPteAddress (PointerPte);
if (PointerPde->u.Hard.Valid == 0) {
MiMakeSystemAddressValidPfn (PointerPte, OldIrql);
}
PfnProto = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
MI_ADD_LOCKED_PAGE_CHARGE (PfnProto);
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
}
recheck:
PteContents = *PointerPte;
// LWFIX: are zero or dzero ptes possible here ?
ASSERT (PteContents.u.Long != 0);
if (PteContents.u.Hard.Valid == 1) {
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
*ApiPage = PageFrameIndex;
*IoPage = DummyPage;
continue;
}
if ((PteContents.u.Soft.Prototype == 0) &&
(PteContents.u.Soft.Transition == 1)) {
//
// The page is in transition. If there is an inpage still in
// progress, wait for it to complete. Reference the PFN and
// then march on.
//
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
if (Pfn1->u4.InPageError) {
//
// There was an in-page read error and there are other
// threads colliding for this page, delay to let the
// other threads complete and then retry.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
if (Pfn1->u3.e1.ReadInProgress) {
// LWFIX - start with temp\aw.c
}
//
// PTE refers to a normal transition PTE.
//
ASSERT ((SPFN_NUMBER)MmAvailablePages >= 0);
if (MmAvailablePages == 0) {
//
// This can only happen if the system is utilizing a hardware
// compression cache. This ensures that only a safe amount
// of the compressed virtual cache is directly mapped so that
// if the hardware gets into trouble, we can bail it out.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
//
// The PFN reference count will be 1 already here if the
// modified writer has begun a write of this page. Otherwise
// it's ordinarily 0.
//
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
*IoPage = DummyPage;
*ApiPage = PageFrameIndex;
continue;
}
// LWFIX: need to handle protos that are now pagefile (or dzero)
// backed - prefetching it from the file here would cause us to lose
// the contents. Note this can happen for session-space images
// as we back modified (ie: for relocation fixups or IAT
// updated) portions from the pagefile. remove the assert below too.
ASSERT (PteContents.u.Soft.Prototype == 1);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (NULL, OldIrql))) {
//
// Had to wait so recheck all state.
//
goto recheck;
}
NumberOfPagesNeedingIo += 1;
//
// Allocate a physical page.
//
PageColor = MI_PAGE_COLOR_VA_PROCESS (
MiGetVirtualAddressMappedByPte (PointerPte),
&CurrentProcess->NextPageColor);
PageFrameIndex = MiRemoveAnyPage (PageColor);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (PointerPte->u.Hard.Valid == 0);
//
// Initialize read-in-progress PFN.
//
MiInitializePfn (PageFrameIndex, PointerPte, 0);
//
// These pieces of MiInitializePfn initialization are overridden
// here as these pages are only going into prototype
// transition and not into any page tables.
//
Pfn1->u3.e1.PrototypePte = 1;
Pfn1->u2.ShareCount -= 1;
ASSERT (Pfn1->u2.ShareCount == 0);
Pfn1->u3.e1.PageLocation = ZeroedPageList;
Pfn1->u3.e2.ReferenceCount -= 1;
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
//
// Initialize the I/O specific fields.
//
Pfn1->u1.Event = &InPageSupport->Event;
Pfn1->u3.e1.ReadInProgress = 1;
ASSERT (Pfn1->u4.InPageError == 0);
//
// Increment the PFN reference count in the control area for
// the subsection.
//
MiReadInfo->ControlArea->NumberOfPfnReferences += 1;
//
// Put the prototype PTE into the transition state.
//
MI_MAKE_TRANSITION_PTE (TempPte,
PageFrameIndex,
PointerPte->u.Soft.Protection,
PointerPte);
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
*IoPage = PageFrameIndex;
*ApiPage = PageFrameIndex;
}
//
// If all the pages were resident, dereference the dummy page references
// now and notify our caller that I/O is not necessary.
//
if (NumberOfPagesNeedingIo == 0) {
ASSERT (DummyPfn1->u3.e2.ReferenceCount > MdlPages);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - MdlPages);
//
// Unlock page containing prototype PTEs.
//
if (PfnProto != NULL) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
}
UNLOCK_PFN (OldIrql);
//
// Return the upfront resident available charge as the
// individual charges have all been made at this point.
//
MI_INCREMENT_RESIDENT_AVAILABLE (ResidentAvailableCharge,
MM_RESAVAIL_FREE_BUILDMDL_EXCESS);
return STATUS_SUCCESS;
}
//
// Carefully trim leading dummy pages.
//
Page = (PPFN_NUMBER)(Mdl + 1);
DummyTrim = 0;
for (i = 0; i < MdlPages - 1; i += 1) {
if (*Page == DummyPage) {
DummyTrim += 1;
Page += 1;
}
else {
break;
}
}
if (DummyTrim != 0) {
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
ASSERT (Mdl->ByteCount != 0);
InPageSupport->ReadOffset.QuadPart += (DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
//
// Shuffle down the PFNs in the MDL.
// Recalculate BasePte to adjust for the shuffle.
//
Pfn1 = MI_PFN_ELEMENT (*Page);
ASSERT (Pfn1->PteAddress->u.Hard.Valid == 0);
ASSERT ((Pfn1->PteAddress->u.Soft.Prototype == 0) &&
(Pfn1->PteAddress->u.Soft.Transition == 1));
InPageSupport->BasePte = Pfn1->PteAddress;
DestinationPage = (PPFN_NUMBER)(Mdl + 1);
do {
*DestinationPage = *Page;
DestinationPage += 1;
Page += 1;
i += 1;
} while (i < MdlPages);
MdlPages -= DummyTrim;
}
//
// Carefully trim trailing dummy pages.
//
ASSERT (MdlPages != 0);
Page = (PPFN_NUMBER)(Mdl + 1) + MdlPages - 1;
if (*Page == DummyPage) {
ASSERT (MdlPages >= 2);
//
// Trim the last page specially as it may be a partial page.
//
Mdl->Size -= sizeof(PFN_NUMBER);
if (BYTE_OFFSET(Mdl->ByteCount) != 0) {
Mdl->ByteCount &= ~(PAGE_SIZE - 1);
}
else {
Mdl->ByteCount -= PAGE_SIZE;
}
ASSERT (Mdl->ByteCount != 0);
DummyPfn1->u3.e2.ReferenceCount -= 1;
//
// Now trim any other trailing pages.
//
Page -= 1;
DummyTrim = 0;
while (Page != ((PPFN_NUMBER)(Mdl + 1))) {
if (*Page != DummyPage) {
break;
}
DummyTrim += 1;
Page -= 1;
}
if (DummyTrim != 0) {
ASSERT (Mdl->Size > (USHORT)(DummyTrim * sizeof(PFN_NUMBER)));
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
}
ASSERT (MdlPages > DummyTrim + 1);
MdlPages -= (DummyTrim + 1);
#if DBG
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
ASSERT (MdlPages == ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa,
Mdl->ByteCount));
#endif
}
//
// If the MDL is not already embedded in the inpage block, see if its
// final size qualifies it - if so, embed it now.
//
if ((Mdl != &InPageSupport->Mdl) &&
(Mdl->ByteCount <= (MM_MAXIMUM_READ_CLUSTER_SIZE + 1) * PAGE_SIZE)){
#if DBG
RtlFillMemoryUlong (&InPageSupport->Page[0],
(MM_MAXIMUM_READ_CLUSTER_SIZE+1) * sizeof (PFN_NUMBER),
0xf1f1f1f1);
#endif
RtlCopyMemory (&InPageSupport->Mdl, Mdl, Mdl->Size);
FreeMdl = Mdl;
Mdl = &InPageSupport->Mdl;
ASSERT (((ULONG_PTR)Mdl & (sizeof(QUAD) - 1)) == 0);
InPageSupport->u1.e1.PrefetchMdlHighBits = ((ULONG_PTR)Mdl >> 3);
}
PFN_NUMBER
NTAPI
MiFindContiguousPages(IN PFN_NUMBER LowestPfn,
IN PFN_NUMBER HighestPfn,
IN PFN_NUMBER BoundaryPfn,
IN PFN_NUMBER SizeInPages,
IN MEMORY_CACHING_TYPE CacheType)
{
PFN_NUMBER Page, PageCount, LastPage, Length, BoundaryMask;
ULONG i = 0;
PMMPFN Pfn1, EndPfn;
KIRQL OldIrql;
PAGED_CODE();
ASSERT(SizeInPages != 0);
//
// Convert the boundary PFN into an alignment mask
//
BoundaryMask = ~(BoundaryPfn - 1);
/* Disable APCs */
KeEnterGuardedRegion();
//
// Loop all the physical memory blocks
//
do
{
//
// Capture the base page and length of this memory block
//
Page = MmPhysicalMemoryBlock->Run[i].BasePage;
PageCount = MmPhysicalMemoryBlock->Run[i].PageCount;
//
// Check how far this memory block will go
//
LastPage = Page + PageCount;
//
// Trim it down to only the PFNs we're actually interested in
//
if ((LastPage - 1) > HighestPfn) LastPage = HighestPfn + 1;
if (Page < LowestPfn) Page = LowestPfn;
//
// Skip this run if it's empty or fails to contain all the pages we need
//
if (!(PageCount) || ((Page + SizeInPages) > LastPage)) continue;
//
// Now scan all the relevant PFNs in this run
//
Length = 0;
for (Pfn1 = MI_PFN_ELEMENT(Page); Page < LastPage; Page++, Pfn1++)
{
//
// If this PFN is in use, ignore it
//
if (MiIsPfnInUse(Pfn1))
{
Length = 0;
continue;
}
//
// If we haven't chosen a start PFN yet and the caller specified an
// alignment, make sure the page matches the alignment restriction
//
if ((!(Length) && (BoundaryPfn)) &&
(((Page ^ (Page + SizeInPages - 1)) & BoundaryMask)))
{
//
// It does not, so bail out
//
continue;
}
//
// Increase the number of valid pages, and check if we have enough
//
if (++Length == SizeInPages)
{
//
// It appears we've amassed enough legitimate pages, rollback
//
Pfn1 -= (Length - 1);
Page -= (Length - 1);
//
// Acquire the PFN lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// Things might've changed for us. Is the page still free?
//
if (MiIsPfnInUse(Pfn1)) break;
//
// So far so good. Is this the last confirmed valid page?
//
if (!--Length)
{
//
// Sanity check that we didn't go out of bounds
//
ASSERT(i != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Loop until all PFN entries have been processed
//
EndPfn = Pfn1 - SizeInPages + 1;
do
{
//
// This PFN is now a used page, set it up
//
MI_SET_USAGE(MI_USAGE_CONTINOUS_ALLOCATION);
MI_SET_PROCESS2("Kernel Driver");
MiUnlinkFreeOrZeroedPage(Pfn1);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u3.e1.StartOfAllocation = 0;
Pfn1->u3.e1.EndOfAllocation = 0;
Pfn1->u3.e1.PrototypePte = 0;
Pfn1->u4.VerifierAllocation = 0;
Pfn1->PteAddress = (PVOID)0xBAADF00D;
//
// Check if this is the last PFN, otherwise go on
//
if (Pfn1 == EndPfn) break;
Pfn1--;
} while (TRUE);
//
// Mark the first and last PFN so we can find them later
//
Pfn1->u3.e1.StartOfAllocation = 1;
(Pfn1 + SizeInPages - 1)->u3.e1.EndOfAllocation = 1;
//
// Now it's safe to let go of the PFN lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// Quick sanity check that the last PFN is consistent
//
EndPfn = Pfn1 + SizeInPages;
ASSERT(EndPfn == MI_PFN_ELEMENT(Page + 1));
//
// Compute the first page, and make sure it's consistent
//
Page = Page - SizeInPages + 1;
ASSERT(Pfn1 == MI_PFN_ELEMENT(Page));
ASSERT(Page != 0);
/* Enable APCs and return the page */
KeLeaveGuardedRegion();
return Page;
}
//
// Keep going. The purpose of this loop is to reconfirm that
// after acquiring the PFN lock these pages are still usable
//
Pfn1++;
Page++;
} while (TRUE);
//
// If we got here, something changed while we hadn't acquired
// the PFN lock yet, so we'll have to restart
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
Length = 0;
}
}
} while (++i != MmPhysicalMemoryBlock->NumberOfRuns);
//
// And if we get here, it means no suitable physical memory runs were found
//
return 0;
}
LOGICAL
MiSetDirtyBit (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte,
IN ULONG PfnHeld
)
/*++
Routine Description:
This routine sets dirty in the specified PTE and the modify bit in the
corresponding PFN element. If any page file space is allocated, it
is deallocated.
Arguments:
FaultingAddress - Supplies the faulting address.
PointerPte - Supplies a pointer to the corresponding valid PTE.
PfnHeld - Supplies TRUE if the PFN lock is already held.
Return Value:
TRUE if action was taken, FALSE if not.
Environment:
Kernel mode, APCs disabled, working set pushlock held.
--*/
{
MMPTE TempPte;
PFN_NUMBER PageFrameIndex;
PMMPFN Pfn1;
//
// The page is NOT copy on write, update the PTE setting both the
// dirty bit and the accessed bit. Note, that as this PTE is in
// the TB, the TB must be flushed.
//
TempPte = *PointerPte;
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte);
//
// This may be a PTE from a rotate physical frame so there may be no
// corresponding PFN for it.
//
if (!MI_IS_PFN (PageFrameIndex)) {
return FALSE;
}
MI_SET_PTE_DIRTY (TempPte);
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte);
//
// Check state of PFN lock and if not held, don't update PFN database.
//
if (PfnHeld) {
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Set the modified field in the PFN database, also, if the physical
// page is currently in a paging file, free up the page file space
// as the contents are now worthless.
//
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
(Pfn1->u3.e1.WriteInProgress == 0)) {
//
// This page is in page file format, deallocate the page file space.
//
MiReleasePageFileSpace (Pfn1->OriginalPte);
//
// Change original PTE to indicate no page file space is reserved,
// otherwise the space will be deallocated when the PTE is
// deleted.
//
Pfn1->OriginalPte.u.Soft.PageFileHigh = 0;
}
MI_SET_MODIFIED (Pfn1, 1, 0x17);
}
//
// The TB entry must be flushed as the valid PTE with the dirty bit clear
// has been fetched into the TB. If it isn't flushed, another fault
// is generated as the dirty bit is not set in the cached TB entry.
//
KeFillEntryTb (FaultingAddress);
return TRUE;
}
LOGICAL
FASTCALL
MiCopyOnWrite (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte
)
/*++
Routine Description:
This routine performs a copy on write operation for the specified
virtual address.
Arguments:
FaultingAddress - Supplies the virtual address which caused the fault.
PointerPte - Supplies the pointer to the PTE which caused the page fault.
Return Value:
Returns TRUE if the page was actually split, FALSE if not.
Environment:
Kernel mode, APCs disabled, working set mutex held.
--*/
{
MMPTE TempPte;
MMPTE TempPte2;
PMMPTE MappingPte;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER NewPageIndex;
PVOID CopyTo;
PVOID CopyFrom;
KIRQL OldIrql;
PMMPFN Pfn1;
PEPROCESS CurrentProcess;
PMMCLONE_BLOCK CloneBlock;
PMMCLONE_DESCRIPTOR CloneDescriptor;
WSLE_NUMBER WorkingSetIndex;
LOGICAL FakeCopyOnWrite;
PMMWSL WorkingSetList;
PVOID SessionSpace;
PLIST_ENTRY NextEntry;
PIMAGE_ENTRY_IN_SESSION Image;
//
// This is called from MmAccessFault, the PointerPte is valid
// and the working set mutex ensures it cannot change state.
//
// Capture the PTE contents to TempPte.
//
TempPte = *PointerPte;
ASSERT (TempPte.u.Hard.Valid == 1);
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Check to see if this is a prototype PTE with copy on write enabled.
//
FakeCopyOnWrite = FALSE;
CurrentProcess = PsGetCurrentProcess ();
CloneBlock = NULL;
if (FaultingAddress >= (PVOID) MmSessionBase) {
WorkingSetList = MmSessionSpace->Vm.VmWorkingSetList;
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
SessionSpace = (PVOID) MmSessionSpace;
MM_SESSION_SPACE_WS_LOCK_ASSERT ();
if (MmSessionSpace->ImageLoadingCount != 0) {
NextEntry = MmSessionSpace->ImageList.Flink;
while (NextEntry != &MmSessionSpace->ImageList) {
Image = CONTAINING_RECORD (NextEntry, IMAGE_ENTRY_IN_SESSION, Link);
if ((FaultingAddress >= Image->Address) &&
(FaultingAddress <= Image->LastAddress)) {
if (Image->ImageLoading) {
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
TempPte.u.Hard.CopyOnWrite = 0;
TempPte.u.Hard.Write = 1;
//
// The page is no longer copy on write, update the PTE
// setting both the dirty bit and the accessed bit.
//
// Even though the page's current backing is the image
// file, the modified writer will convert it to
// pagefile backing when it notices the change later.
//
MI_SET_PTE_DIRTY (TempPte);
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte);
//
// The TB entry must be flushed as the valid PTE with
// the dirty bit clear has been fetched into the TB. If
// it isn't flushed, another fault is generated as the
// dirty bit is not set in the cached TB entry.
//
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
return FALSE;
}
break;
}
NextEntry = NextEntry->Flink;
}
}
}
else {
WorkingSetList = MmWorkingSetList;
SessionSpace = NULL;
//
// If a fork operation is in progress, block until the fork is
// completed, then retry the whole operation as the state of
// everything may have changed between when the mutexes were
// released and reacquired.
//
if (CurrentProcess->ForkInProgress != NULL) {
if (MiWaitForForkToComplete (CurrentProcess) == TRUE) {
return FALSE;
}
}
if (TempPte.u.Hard.CopyOnWrite == 0) {
//
// This is a fork page which is being made private in order
// to change the protection of the page.
// Do not make the page writable.
//
FakeCopyOnWrite = TRUE;
}
}
WorkingSetIndex = MiLocateWsle (FaultingAddress,
WorkingSetList,
Pfn1->u1.WsIndex,
FALSE);
//
// The page must be copied into a new page.
//
LOCK_PFN (OldIrql);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (SessionSpace != NULL ? HYDRA_PROCESS : CurrentProcess, OldIrql))) {
//
// A wait operation was performed to obtain an available
// page and the working set mutex and PFN lock have
// been released and various things may have changed for
// the worse. Rather than examine all the conditions again,
// return and if things are still proper, the fault will
// be taken again.
//
UNLOCK_PFN (OldIrql);
return FALSE;
}
//
// This must be a prototype PTE. Perform the copy on write.
//
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
//
// A page is being copied and made private, the global state of
// the shared page needs to be updated at this point on certain
// hardware. This is done by ORing the dirty bit into the modify bit in
// the PFN element.
//
// Note that a session page cannot be dirty (no POSIX-style forking is
// supported for these drivers).
//
if (SessionSpace != NULL) {
ASSERT ((TempPte.u.Hard.Valid == 1) && (TempPte.u.Hard.Write == 0));
ASSERT (!MI_IS_PTE_DIRTY (TempPte));
NewPageIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_SESSION(MmSessionSpace));
}
else {
MI_CAPTURE_DIRTY_BIT_TO_PFN (PointerPte, Pfn1);
CloneBlock = (PMMCLONE_BLOCK) Pfn1->PteAddress;
//
// Get a new page with the same color as this page.
//
NewPageIndex = MiRemoveAnyPage (
MI_PAGE_COLOR_PTE_PROCESS(PageFrameIndex,
&CurrentProcess->NextPageColor));
}
MiInitializeCopyOnWritePfn (NewPageIndex,
PointerPte,
WorkingSetIndex,
WorkingSetList);
UNLOCK_PFN (OldIrql);
InterlockedIncrement (&KeGetCurrentPrcb ()->MmCopyOnWriteCount);
CopyFrom = PAGE_ALIGN (FaultingAddress);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
NewPageIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
if (Pfn1->u3.e1.CacheAttribute == MiNonCached) {
MI_DISABLE_CACHING (TempPte2);
}
else if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined) {
MI_SET_PTE_WRITE_COMBINE (TempPte2);
}
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
CopyTo = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
CopyTo = MiMapPageInHyperSpace (CurrentProcess,
NewPageIndex,
&OldIrql);
}
KeCopyPage (CopyTo, CopyFrom);
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, CopyTo, OldIrql);
}
if (!FakeCopyOnWrite) {
//
// If the page was really a copy on write page, make it
// accessed, dirty and writable. Also, clear the copy-on-write
// bit in the PTE.
//
MI_SET_PTE_DIRTY (TempPte);
TempPte.u.Hard.Write = 1;
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
TempPte.u.Hard.CopyOnWrite = 0;
}
//
// Regardless of whether the page was really a copy on write,
// the frame field of the PTE must be updated.
//
TempPte.u.Hard.PageFrameNumber = NewPageIndex;
//
// If the modify bit is set in the PFN database for the
// page, the data cache must be flushed. This is due to the
// fact that this process may have been cloned and the cache
// still contains stale data destined for the page we are
// going to remove.
//
ASSERT (TempPte.u.Hard.Valid == 1);
MI_WRITE_VALID_PTE_NEW_PAGE (PointerPte, TempPte);
//
// Flush the TB entry for this page.
//
if (SessionSpace == NULL) {
MI_FLUSH_SINGLE_TB (FaultingAddress, FALSE);
//
// Increment the number of private pages.
//
CurrentProcess->NumberOfPrivatePages += 1;
}
else {
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
}
//
// Decrement the share count for the page which was copied
// as this PTE no longer refers to it.
//
LOCK_PFN (OldIrql);
MiDecrementShareCount (Pfn1, PageFrameIndex);
if (SessionSpace == NULL) {
CloneDescriptor = MiLocateCloneAddress (CurrentProcess,
(PVOID)CloneBlock);
if (CloneDescriptor != NULL) {
//
// Decrement the reference count for the clone block,
// note that this could release and reacquire the mutexes.
//
MiDecrementCloneBlockReference (CloneDescriptor,
CloneBlock,
CurrentProcess,
NULL,
OldIrql);
}
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
VOID
MiProcessValidPteList (
IN PMMPTE *ValidPteList,
IN ULONG Count
)
/*++
Routine Description:
This routine flushes the specified range of valid PTEs.
Arguments:
ValidPteList - Supplies a pointer to an array of PTEs to flush.
Count - Supplies the count of the number of elements in the array.
Return Value:
none.
Environment:
Kernel mode, APCs disabled, WorkingSetMutex and AddressCreation mutexes
held.
--*/
{
ULONG i;
MMPTE_FLUSH_LIST PteFlushList;
MMPTE PteContents;
PMMPFN Pfn1;
PMMPFN Pfn2;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER PageTableFrameIndex;
KIRQL OldIrql;
i = 0;
PteFlushList.Count = Count;
if (Count < MM_MAXIMUM_FLUSH_COUNT) {
do {
PteFlushList.FlushVa[i] =
MiGetVirtualAddressMappedByPte (ValidPteList[i]);
i += 1;
} while (i != Count);
i = 0;
}
LOCK_PFN (OldIrql);
do {
PteContents = *ValidPteList[i];
ASSERT (PteContents.u.Hard.Valid == 1);
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE(&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Decrement the share and valid counts of the page table
// page which maps this PTE.
//
PageTableFrameIndex = Pfn1->u4.PteFrame;
Pfn2 = MI_PFN_ELEMENT (PageTableFrameIndex);
MiDecrementShareCountInline (Pfn2, PageTableFrameIndex);
MI_SET_PFN_DELETED (Pfn1);
//
// Decrement the share count for the physical page. As the page
// is private it will be put on the free list.
//
MiDecrementShareCount (Pfn1, PageFrameIndex);
MI_WRITE_INVALID_PTE (ValidPteList[i], MmDecommittedPte);
i += 1;
} while (i != Count);
MiFlushPteList (&PteFlushList);
UNLOCK_PFN (OldIrql);
return;
}
ULONG
MiDecommitPages (
IN PVOID StartingAddress,
IN PMMPTE EndingPte,
IN PEPROCESS Process,
IN PMMVAD_SHORT Vad
)
/*++
Routine Description:
This routine decommits the specified range of pages.
Arguments:
StartingAddress - Supplies the starting address of the range.
EndingPte - Supplies the ending PTE of the range.
Process - Supplies the current process.
Vad - Supplies the virtual address descriptor which describes the range.
Return Value:
Value to reduce commitment by for the VAD.
Environment:
Kernel mode, APCs disabled, AddressCreation mutex held.
--*/
{
PMMPTE PointerPde;
PMMPTE PointerPte;
PVOID Va;
ULONG CommitReduction;
PMMPTE CommitLimitPte;
KIRQL OldIrql;
PMMPTE ValidPteList[MM_VALID_PTE_SIZE];
ULONG count;
WSLE_NUMBER WorkingSetIndex;
PMMPFN Pfn1;
PMMPFN Pfn2;
WSLE_NUMBER Entry;
MMWSLENTRY Locked;
MMPTE PteContents;
PFN_NUMBER PageTableFrameIndex;
PVOID UsedPageTableHandle;
PETHREAD CurrentThread;
count = 0;
CommitReduction = 0;
if (Vad->u.VadFlags.MemCommit) {
CommitLimitPte = MiGetPteAddress (MI_VPN_TO_VA (Vad->EndingVpn));
}
else {
CommitLimitPte = NULL;
}
//
// Decommit each page by setting the PTE to be explicitly
// decommitted. The PTEs cannot be deleted all at once as
// this would set the PTEs to zero which would auto-evaluate
// as committed if referenced by another thread when a page
// table page is being in-paged.
//
PointerPde = MiGetPdeAddress (StartingAddress);
PointerPte = MiGetPteAddress (StartingAddress);
Va = StartingAddress;
//
// Loop through all the PDEs which map this region and ensure that
// they exist. If they don't exist create them by touching a
// PTE mapped by the PDE.
//
CurrentThread = PsGetCurrentThread ();
LOCK_WS_UNSAFE (CurrentThread, Process);
MiMakePdeExistAndMakeValid (PointerPde, Process, MM_NOIRQL);
while (PointerPte <= EndingPte) {
if (MiIsPteOnPdeBoundary (PointerPte)) {
PointerPde = MiGetPdeAddress (Va);
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
MiMakePdeExistAndMakeValid (PointerPde, Process, MM_NOIRQL);
}
//
// The working set lock is held. No PTEs can go from
// invalid to valid or valid to invalid. Transition
// PTEs can go from transition to pagefile.
//
PteContents = *PointerPte;
if (PteContents.u.Long != 0) {
if (PointerPte->u.Long == MmDecommittedPte.u.Long) {
//
// This PTE is already decommitted.
//
CommitReduction += 1;
}
else {
Process->NumberOfPrivatePages -= 1;
if (PteContents.u.Hard.Valid == 1) {
//
// Make sure this is not a forked PTE.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
if (Pfn1->u3.e1.PrototypePte) {
//
// MiDeletePte may release both the working set pushlock
// and the PFN lock so the valid PTE list must be
// processed now.
//
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL,
OldIrql);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
else {
//
// PTE is valid, process later when PFN lock is held.
//
if (count == MM_VALID_PTE_SIZE) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
ValidPteList[count] = PointerPte;
count += 1;
//
// Remove address from working set list.
//
WorkingSetIndex = Pfn1->u1.WsIndex;
ASSERT (PAGE_ALIGN(MmWsle[WorkingSetIndex].u1.Long) ==
Va);
//
// Check to see if this entry is locked in the
// working set or locked in memory.
//
Locked = MmWsle[WorkingSetIndex].u1.e1;
MiRemoveWsle (WorkingSetIndex, MmWorkingSetList);
//
// Add this entry to the list of free working set
// entries and adjust the working set count.
//
MiReleaseWsle (WorkingSetIndex, &Process->Vm);
if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {
//
// This entry is locked.
//
MmWorkingSetList->FirstDynamic -= 1;
if (WorkingSetIndex != MmWorkingSetList->FirstDynamic) {
Entry = MmWorkingSetList->FirstDynamic;
ASSERT (MmWsle[Entry].u1.e1.Valid);
MiSwapWslEntries (Entry,
WorkingSetIndex,
&Process->Vm,
FALSE);
}
}
MI_SET_PTE_IN_WORKING_SET (PointerPte, 0);
}
}
else if (PteContents.u.Soft.Prototype) {
//
// This is a forked PTE, just delete it.
//
// MiDeletePte may release both the working set pushlock
// and the PFN lock so the valid PTE list must be
// processed now.
//
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL,
OldIrql);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
else if (PteContents.u.Soft.Transition == 1) {
//
// Transition PTE, get the PFN database lock
// and reprocess this one.
//
LOCK_PFN (OldIrql);
PteContents = *PointerPte;
if (PteContents.u.Soft.Transition == 1) {
//
// PTE is still in transition, delete it.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
MI_SET_PFN_DELETED (Pfn1);
PageTableFrameIndex = Pfn1->u4.PteFrame;
Pfn2 = MI_PFN_ELEMENT (PageTableFrameIndex);
MiDecrementShareCountInline (Pfn2, PageTableFrameIndex);
//
// Check the reference count for the page, if the
// reference count is zero, move the page to the
// free list, if the reference count is not zero,
// ignore this page. When the reference count
// goes to zero, it will be placed on the free list.
//
if (Pfn1->u3.e2.ReferenceCount == 0) {
MiUnlinkPageFromList (Pfn1);
MiReleasePageFileSpace (Pfn1->OriginalPte);
MiInsertPageInFreeList (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE(&PteContents));
}
}
else {
//
// Page MUST be in page file format!
//
ASSERT (PteContents.u.Soft.Valid == 0);
ASSERT (PteContents.u.Soft.Prototype == 0);
ASSERT (PteContents.u.Soft.PageFileHigh != 0);
MiReleasePageFileSpace (PteContents);
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
UNLOCK_PFN (OldIrql);
}
else {
//
// Must be demand zero or paging file format.
//
if (PteContents.u.Soft.PageFileHigh != 0) {
LOCK_PFN (OldIrql);
MiReleasePageFileSpace (PteContents);
UNLOCK_PFN (OldIrql);
}
else {
//
// Don't subtract out the private page count for
// a demand zero page.
//
Process->NumberOfPrivatePages += 1;
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
}
}
else {
//
// The PTE is already zero.
//
//
// Increment the count of non-zero page table entries for this
// page table and the number of private pages for the process.
//
UsedPageTableHandle = MI_GET_USED_PTES_HANDLE (Va);
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableHandle);
if (PointerPte > CommitLimitPte) {
//
// PTE is not committed.
//
CommitReduction += 1;
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
PointerPte += 1;
Va = (PVOID)((PCHAR)Va + PAGE_SIZE);
}
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
}
UNLOCK_WS_UNSAFE (CurrentThread, Process);
return CommitReduction;
}
ULONG
MiDecommitPages (
IN PVOID StartingAddress,
IN PMMPTE EndingPte,
IN PEPROCESS Process,
IN PMMVAD_SHORT Vad
)
/*++
Routine Description:
This routine decommits the specficed range of pages.
Arguments:
StartingAddress - Supplies the starting address of the range.
EndingPte - Supplies the ending PTE of the range.
Process - Supplies the current process.
Vad - Supplies the virtual address descriptor which describes the range.
Return Value:
Value to reduce commitment by for the VAD.
Environment:
Kernel mode, APCs disable, WorkingSetMutex and AddressCreation mutexes
held.
--*/
{
PMMPTE PointerPde;
PMMPTE PointerPte;
PVOID Va;
ULONG PdeOffset;
ULONG CommitReduction = 0;
PMMPTE CommitLimitPte;
KIRQL OldIrql;
PMMPTE ValidPteList[MM_VALID_PTE_SIZE];
ULONG count = 0;
ULONG WorkingSetIndex;
PMMPFN Pfn1;
PMMPFN Pfn2;
PVOID SwapVa;
ULONG Entry;
MMWSLENTRY Locked;
MMPTE PteContents;
if (Vad->u.VadFlags.MemCommit) {
CommitLimitPte = MiGetPteAddress (Vad->EndingVa);
} else {
CommitLimitPte = NULL;
}
//
// Decommit each page by setting the PTE to be explicitly
// decommitted. The PTEs cannot be deleted all at once as
// this would set the PTEs to zero which would auto-evaluate
// as committed if referenced by another thread when a page
// table page is being in-paged.
//
PointerPde = MiGetPdeAddress (StartingAddress);
PointerPte = MiGetPteAddress (StartingAddress);
Va = StartingAddress;
PdeOffset = MiGetPdeOffset (Va);
//
// Loop through all the PDEs which map this region and ensure that
// they exist. If they don't exist create them by touching a
// PTE mapped by the PDE.
//
//
// Get the PFN mutex so the MiDeletePte can be called.
//
MiMakePdeExistAndMakeValid(PointerPde, Process, FALSE);
while (PointerPte <= EndingPte) {
if (((ULONG)PointerPte & (PAGE_SIZE - 1)) == 0) {
PdeOffset = MiGetPdeOffset (Va);
PointerPde = MiGetPdeAddress (Va);
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
MiMakePdeExistAndMakeValid(PointerPde, Process, FALSE);
}
//
// The working set lock is held. No PTEs can go from
// invalid to valid or valid to invalid. Transition
// PTEs can go from transition to pagefile.
//
PteContents = *PointerPte;
if (PteContents.u.Long != 0) {
if (PointerPte->u.Long == MmDecommittedPte.u.Long) {
//
// This PTE is already decommitted.
//
CommitReduction += 1;
} else {
Process->NumberOfPrivatePages -= 1;
if (PteContents.u.Hard.Valid == 1) {
//
// Make sure this is not a forked PTE.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
if (Pfn1->u3.e1.PrototypePte) {
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
*PointerPte = MmDecommittedPte;
} else {
//
// Pte is valid, process later when PFN lock is held.
//
if (count == MM_VALID_PTE_SIZE) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
ValidPteList[count] = PointerPte;
count += 1;
//
// Remove address from working set list.
//
WorkingSetIndex = Pfn1->u1.WsIndex;
ASSERT (PAGE_ALIGN(MmWsle[WorkingSetIndex].u1.Long) ==
Va);
//
// Check to see if this entry is locked in the working set
// or locked in memory.
//
Locked = MmWsle[WorkingSetIndex].u1.e1;
MiRemoveWsle (WorkingSetIndex, MmWorkingSetList);
//
// Add this entry to the list of free working set entries
// and adjust the working set count.
//
MiReleaseWsle (WorkingSetIndex, &Process->Vm);
if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {
//
// This entry is locked.
//
MmWorkingSetList->FirstDynamic -= 1;
if (WorkingSetIndex != MmWorkingSetList->FirstDynamic) {
SwapVa = MmWsle[MmWorkingSetList->FirstDynamic].u1.VirtualAddress;
SwapVa = PAGE_ALIGN (SwapVa);
Pfn2 = MI_PFN_ELEMENT (
MiGetPteAddress (SwapVa)->u.Hard.PageFrameNumber);
Entry = MiLocateWsle (SwapVa,
MmWorkingSetList,
Pfn2->u1.WsIndex);
MiSwapWslEntries (Entry,
WorkingSetIndex,
&Process->Vm);
}
}
}
} else if (PteContents.u.Soft.Prototype) {
//
// This is a forked PTE, just delete it.
//
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
*PointerPte = MmDecommittedPte;
} else if (PteContents.u.Soft.Transition == 1) {
//
// Transition PTE, get the PFN database lock
// and reprocess this one.
//
LOCK_PFN (OldIrql);
PteContents = *PointerPte;
if (PteContents.u.Soft.Transition == 1) {
//
// PTE is still in transition, delete it.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
MI_SET_PFN_DELETED (Pfn1);
MiDecrementShareCount (Pfn1->PteFrame);
//
// Check the reference count for the page, if the
// reference count is zero, move the page to the
// free list, if the reference count is not zero,
// ignore this page. When the refernce count
// goes to zero, it will be placed on the free list.
//
if (Pfn1->u3.e2.ReferenceCount == 0) {
MiUnlinkPageFromList (Pfn1);
MiReleasePageFileSpace (Pfn1->OriginalPte);
MiInsertPageInList (MmPageLocationList[FreePageList],
PteContents.u.Trans.PageFrameNumber);
}
*PointerPte = MmDecommittedPte;
} else {
//
// Page MUST be in page file format!
//
ASSERT (PteContents.u.Soft.Valid == 0);
ASSERT (PteContents.u.Soft.Prototype == 0);
ASSERT (PteContents.u.Soft.PageFileHigh != 0);
MiReleasePageFileSpace (PteContents);
*PointerPte = MmDecommittedPte;
}
UNLOCK_PFN (OldIrql);
} else {
//
// Must be demand zero or paging file format.
//
if (PteContents.u.Soft.PageFileHigh != 0) {
LOCK_PFN (OldIrql);
MiReleasePageFileSpace (PteContents);
UNLOCK_PFN (OldIrql);
} else {
//
// Don't subtract out the private page count for
// a demand zero page.
//
Process->NumberOfPrivatePages += 1;
}
*PointerPte = MmDecommittedPte;
}
}
} else {
//
// The PTE is already zero.
//
//
// Increment the count of non-zero page table entires for this
// page table and the number of private pages for the process.
//
MmWorkingSetList->UsedPageTableEntries[PdeOffset] += 1;
if (PointerPte > CommitLimitPte) {
//
// Pte is not committed.
//
CommitReduction += 1;
}
*PointerPte = MmDecommittedPte;
}
PointerPte += 1;
Va = (PVOID)((ULONG)Va + PAGE_SIZE);
}
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
}
return CommitReduction;
}
VOID
MiZeroPageFile (
IN PVOID Context
)
/*++
Routine Description:
This routine zeroes all inactive pagefile blocks in the specified paging
file.
Arguments:
Context - Supplies the information on which pagefile to zero and a zeroed
page to use for the I/O.
Return Value:
Returns TRUE on success, FALSE on failure.
Environment:
Kernel mode, the caller must lock down PAGELK.
--*/
{
PFN_NUMBER MaxPagesToWrite;
PMMPFN Pfn1;
PPFN_NUMBER Page;
PFN_NUMBER MdlHack[(sizeof(MDL)/sizeof(PFN_NUMBER)) + MM_MAXIMUM_WRITE_CLUSTER];
PMDL Mdl;
NTSTATUS Status;
KEVENT IoEvent;
IO_STATUS_BLOCK IoStatus;
KIRQL OldIrql;
LARGE_INTEGER StartingOffset;
ULONG count;
ULONG i;
PFN_NUMBER first;
ULONG write;
PKEVENT AllDone;
SIZE_T NumberOfBytes;
PMMPAGING_FILE PagingFile;
PFN_NUMBER ZeroedPageFrame;
PMM_ZERO_PAGEFILE_CONTEXT ZeroContext;
ZeroContext = (PMM_ZERO_PAGEFILE_CONTEXT) Context;
PagingFile = ZeroContext->PagingFile;
ZeroedPageFrame = ZeroContext->ZeroedPageFrame;
AllDone = ZeroContext->AllDone;
ExFreePool (Context);
NumberOfBytes = MmModifiedWriteClusterSize << PAGE_SHIFT;
MaxPagesToWrite = NumberOfBytes >> PAGE_SHIFT;
Mdl = (PMDL) MdlHack;
Page = (PPFN_NUMBER)(Mdl + 1);
KeInitializeEvent (&IoEvent, NotificationEvent, FALSE);
MmInitializeMdl (Mdl, NULL, PAGE_SIZE);
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
Mdl->StartVa = NULL;
i = 0;
Page = (PPFN_NUMBER)(Mdl + 1);
for (i = 0; i < MaxPagesToWrite; i += 1) {
*Page = ZeroedPageFrame;
Page += 1;
}
count = 0;
write = FALSE;
SATISFY_OVERZEALOUS_COMPILER (first = 0);
LOCK_PFN (OldIrql);
for (i = 1; i < PagingFile->Size; i += 1) {
if (RtlCheckBit (PagingFile->Bitmap, (ULONG) i) == 0) {
//
// Claim the pagefile location as the modified writer
// may already be scanning.
//
RtlSetBit (PagingFile->Bitmap, (ULONG) i);
if (count == 0) {
first = i;
}
count += 1;
if ((count == MaxPagesToWrite) || (i == PagingFile->Size - 1)) {
write = TRUE;
}
}
else {
if (count != 0) {
//
// Issue a write.
//
write = TRUE;
}
}
if (write) {
UNLOCK_PFN (OldIrql);
StartingOffset.QuadPart = (LONGLONG)first << PAGE_SHIFT;
Mdl->ByteCount = count << PAGE_SHIFT;
KeClearEvent (&IoEvent);
Status = IoSynchronousPageWrite (PagingFile->File,
Mdl,
&StartingOffset,
&IoEvent,
&IoStatus);
//
// Ignore all I/O failures - there is nothing that can
// be done at this point.
//
if (!NT_SUCCESS (Status)) {
KeSetEvent (&IoEvent, 0, FALSE);
}
Status = KeWaitForSingleObject (&IoEvent,
WrPageOut,
KernelMode,
FALSE,
(PLARGE_INTEGER)&MmTwentySeconds);
if (Status == STATUS_TIMEOUT) {
//
// The write did not complete in 20 seconds, assume
// that the file systems are hung and return an error.
//
// Note the zero page (and any MDL system virtual address a
// driver may have created) is leaked because we don't know
// what the filesystem or storage stack might (still) be
// doing to them.
//
Pfn1 = MI_PFN_ELEMENT (ZeroedPageFrame);
LOCK_PFN (OldIrql);
//
// Increment the reference count on the zeroed page to ensure
// it is never freed.
//
InterlockedIncrementPfn ((PSHORT)&Pfn1->u3.e2.ReferenceCount);
RtlClearBits (PagingFile->Bitmap, (ULONG) first, count);
break;
}
if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {
MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
}
write = FALSE;
LOCK_PFN (OldIrql);
RtlClearBits (PagingFile->Bitmap, (ULONG) first, count);
count = 0;
}
}
UNLOCK_PFN (OldIrql);
KeSetEvent (AllDone, 0, FALSE);
return;
}
BOOLEAN
MiShutdownSystem (
VOID
)
/*++
Routine Description:
This function performs the shutdown of memory management. This
is accomplished by writing out all modified pages which are
destined for files other than the paging file.
All processes have already been killed, the registry shutdown and
shutdown IRPs already sent. On return from this phase all mapped
file data must be flushed and the unused segment list emptied.
This releases all the Mm references to file objects, allowing many
drivers (especially the network) to unload.
Arguments:
None.
Return Value:
TRUE if the pages were successfully written, FALSE otherwise.
--*/
{
SIZE_T ImportListSize;
PLOAD_IMPORTS ImportList;
PLOAD_IMPORTS ImportListNonPaged;
PLIST_ENTRY NextEntry;
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
PFN_NUMBER ModifiedPage;
PMMPFN Pfn1;
PSUBSECTION Subsection;
PCONTROL_AREA ControlArea;
PPFN_NUMBER Page;
PFILE_OBJECT FilePointer;
ULONG ConsecutiveFileLockFailures;
PFN_NUMBER MdlHack[(sizeof(MDL)/sizeof(PFN_NUMBER)) + MM_MAXIMUM_WRITE_CLUSTER];
PMDL Mdl;
NTSTATUS Status;
KEVENT IoEvent;
IO_STATUS_BLOCK IoStatus;
KIRQL OldIrql;
LARGE_INTEGER StartingOffset;
ULONG count;
ULONG i;
//
// Don't do this more than once.
//
if (MmSystemShutdown == 0) {
Mdl = (PMDL) MdlHack;
Page = (PPFN_NUMBER)(Mdl + 1);
KeInitializeEvent (&IoEvent, NotificationEvent, FALSE);
MmInitializeMdl (Mdl, NULL, PAGE_SIZE);
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
MmLockPageableSectionByHandle (ExPageLockHandle);
LOCK_PFN (OldIrql);
ModifiedPage = MmModifiedPageListHead.Flink;
while (ModifiedPage != MM_EMPTY_LIST) {
//
// There are modified pages.
//
Pfn1 = MI_PFN_ELEMENT (ModifiedPage);
if (Pfn1->OriginalPte.u.Soft.Prototype == 1) {
//
// This page is destined for a file.
//
Subsection = MiGetSubsectionAddress (&Pfn1->OriginalPte);
ControlArea = Subsection->ControlArea;
if ((!ControlArea->u.Flags.Image) &&
(!ControlArea->u.Flags.NoModifiedWriting)) {
MiUnlinkPageFromList (Pfn1);
//
// Issue the write.
//
MI_SET_MODIFIED (Pfn1, 0, 0x28);
//
// Up the reference count for the physical page as there
// is I/O in progress.
//
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
*Page = ModifiedPage;
ControlArea->NumberOfMappedViews += 1;
ControlArea->NumberOfPfnReferences += 1;
UNLOCK_PFN (OldIrql);
StartingOffset.QuadPart = MiStartingOffset (Subsection,
Pfn1->PteAddress);
Mdl->StartVa = NULL;
ConsecutiveFileLockFailures = 0;
FilePointer = ControlArea->FilePointer;
retry:
KeClearEvent (&IoEvent);
Status = FsRtlAcquireFileForCcFlushEx (FilePointer);
if (NT_SUCCESS(Status)) {
Status = IoSynchronousPageWrite (FilePointer,
Mdl,
&StartingOffset,
&IoEvent,
&IoStatus);
//
// Release the file we acquired.
//
FsRtlReleaseFileForCcFlush (FilePointer);
}
if (!NT_SUCCESS(Status)) {
//
// Only try the request more than once if the
// filesystem said it had a deadlock.
//
if (Status == STATUS_FILE_LOCK_CONFLICT) {
ConsecutiveFileLockFailures += 1;
if (ConsecutiveFileLockFailures < 5) {
KeDelayExecutionThread (KernelMode,
FALSE,
(PLARGE_INTEGER)&MmShortTime);
goto retry;
}
goto wait_complete;
}
//
// Ignore all I/O failures - there is nothing that
// can be done at this point.
//
KeSetEvent (&IoEvent, 0, FALSE);
}
Status = KeWaitForSingleObject (&IoEvent,
WrPageOut,
KernelMode,
FALSE,
(PLARGE_INTEGER)&MmTwentySeconds);
wait_complete:
if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {
MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
}
if (Status == STATUS_TIMEOUT) {
//
// The write did not complete in 20 seconds, assume
// that the file systems are hung and return an
// error.
//
LOCK_PFN (OldIrql);
MI_SET_MODIFIED (Pfn1, 1, 0xF);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (Pfn1);
ControlArea->NumberOfMappedViews -= 1;
ControlArea->NumberOfPfnReferences -= 1;
//
// This routine returns with the PFN lock released!
//
MiCheckControlArea (ControlArea, OldIrql);
MmUnlockPageableImageSection (ExPageLockHandle);
return FALSE;
}
LOCK_PFN (OldIrql);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (Pfn1);
ControlArea->NumberOfMappedViews -= 1;
ControlArea->NumberOfPfnReferences -= 1;
//
// This routine returns with the PFN lock released!
//
MiCheckControlArea (ControlArea, OldIrql);
LOCK_PFN (OldIrql);
//
// Restart scan at the front of the list.
//
ModifiedPage = MmModifiedPageListHead.Flink;
continue;
}
}
ModifiedPage = Pfn1->u1.Flink;
}
UNLOCK_PFN (OldIrql);
//
// Indicate to the modified page writer that the system has
// shutdown.
//
MmSystemShutdown = 1;
//
// Check to see if the paging file should be overwritten.
// Only free blocks are written.
//
if (MmZeroPageFile) {
MiZeroAllPageFiles ();
}
MmUnlockPageableImageSection (ExPageLockHandle);
}
if (PoCleanShutdownEnabled ()) {
//
// Empty the unused segment list.
//
LOCK_PFN (OldIrql);
MmUnusedSegmentForceFree = (ULONG)-1;
KeSetEvent (&MmUnusedSegmentCleanup, 0, FALSE);
//
// Give it 5 seconds to empty otherwise assume the filesystems are
// hung and march on.
//
for (count = 0; count < 500; count += 1) {
if (IsListEmpty(&MmUnusedSegmentList)) {
break;
}
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode,
FALSE,
(PLARGE_INTEGER)&MmShortTime);
LOCK_PFN (OldIrql);
#if DBG
if (count == 400) {
//
// Everything should have been flushed by now. Give the
// filesystem team a chance to debug this on checked builds.
//
ASSERT (FALSE);
}
#endif
//
// Resignal if needed in case more closed file objects triggered
// additional entries.
//
if (MmUnusedSegmentForceFree == 0) {
MmUnusedSegmentForceFree = (ULONG)-1;
KeSetEvent (&MmUnusedSegmentCleanup, 0, FALSE);
}
}
UNLOCK_PFN (OldIrql);
//
// Get rid of any paged pool references as they will be illegal
// by the time MmShutdownSystem is called again since the filesystems
// will have shutdown.
//
KeWaitForSingleObject (&MmSystemLoadLock,
WrVirtualMemory,
KernelMode,
FALSE,
(PLARGE_INTEGER)NULL);
NextEntry = PsLoadedModuleList.Flink;
while (NextEntry != &PsLoadedModuleList) {
DataTableEntry = CONTAINING_RECORD (NextEntry,
KLDR_DATA_TABLE_ENTRY,
InLoadOrderLinks);
ImportList = (PLOAD_IMPORTS)DataTableEntry->LoadedImports;
if ((ImportList != (PVOID)LOADED_AT_BOOT) &&
(ImportList != (PVOID)NO_IMPORTS_USED) &&
(!SINGLE_ENTRY(ImportList))) {
ImportListSize = ImportList->Count * sizeof(PVOID) + sizeof(SIZE_T);
ImportListNonPaged = (PLOAD_IMPORTS) ExAllocatePoolWithTag (NonPagedPool,
ImportListSize,
'TDmM');
if (ImportListNonPaged != NULL) {
RtlCopyMemory (ImportListNonPaged, ImportList, ImportListSize);
ExFreePool (ImportList);
DataTableEntry->LoadedImports = ImportListNonPaged;
}
else {
//
// Don't bother with the clean shutdown at this point.
//
PopShutdownCleanly = FALSE;
break;
}
}
//
// Free the full DLL name as it is pageable.
//
if (DataTableEntry->FullDllName.Buffer != NULL) {
ExFreePool (DataTableEntry->FullDllName.Buffer);
DataTableEntry->FullDllName.Buffer = NULL;
}
NextEntry = NextEntry->Flink;
}
KeReleaseMutant (&MmSystemLoadLock, 1, FALSE, FALSE);
//
// Close all the pagefile handles, note we still have an object
// reference to each keeping the underlying object resident.
// At the end of Phase1 shutdown we'll release those references
// to trigger the storage stack unload. The handle close must be
// done here however as it will reference pageable structures.
//
for (i = 0; i < MmNumberOfPagingFiles; i += 1) {
//
// Free each pagefile name now as it resides in paged pool and
// may need to be inpaged to be freed. Since the paging files
// are going to be shutdown shortly, now is the time to access
// pageable stuff and get rid of it. Zeroing the buffer pointer
// is sufficient as the only accesses to this are from the
// try-except-wrapped GetSystemInformation APIs and all the
// user processes are gone already.
//
ASSERT (MmPagingFile[i]->PageFileName.Buffer != NULL);
ExFreePool (MmPagingFile[i]->PageFileName.Buffer);
MmPagingFile[i]->PageFileName.Buffer = NULL;
ZwClose (MmPagingFile[i]->FileHandle);
}
}
return TRUE;
}
NTSTATUS
MmRemovePhysicalMemory (
IN PPHYSICAL_ADDRESS StartAddress,
IN OUT PLARGE_INTEGER NumberOfBytes
)
/*++
Routine Description:
This routine attempts to remove the specified physical address range
from the system.
Arguments:
StartAddress - Supplies the starting physical address.
NumberOfBytes - Supplies a pointer to the number of bytes being removed.
Return Value:
NTSTATUS.
Environment:
Kernel mode. PASSIVE level. No locks held.
--*/
{
ULONG i;
ULONG Additional;
PFN_NUMBER Page;
PFN_NUMBER LastPage;
PFN_NUMBER OriginalLastPage;
PFN_NUMBER start;
PFN_NUMBER PagesReleased;
PMMPFN Pfn1;
PMMPFN StartPfn;
PMMPFN EndPfn;
KIRQL OldIrql;
PFN_NUMBER StartPage;
PFN_NUMBER EndPage;
PFN_COUNT NumberOfPages;
SPFN_NUMBER MaxPages;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER RemovedPages;
LOGICAL Inserted;
NTSTATUS Status;
PMMPTE PointerPte;
PMMPTE EndPte;
PVOID VirtualAddress;
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
PPHYSICAL_MEMORY_RUN NewRun;
LOGICAL PfnDatabaseIsPhysical;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
//
// The system must be configured for dynamic memory addition. This is
// not strictly required to remove the memory, but it's better to check
// for it now under the assumption that the administrator is probably
// going to want to add this range of memory back in - better to give
// the error now and refuse the removal than to refuse the addition
// later.
//
if (MmDynamicPfn == FALSE) {
return STATUS_NOT_SUPPORTED;
}
PfnDatabaseIsPhysical = TRUE;
}
else {
PfnDatabaseIsPhysical = FALSE;
}
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
EndPage = StartPage + NumberOfPages;
if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
//
// Truncate the request into something that can be mapped by the PFN
// database.
//
EndPage = MmHighestPossiblePhysicalPage + 1;
NumberOfPages = (PFN_COUNT)(EndPage - StartPage);
}
//
// The range cannot wrap.
//
if (StartPage >= EndPage) {
return STATUS_INVALID_PARAMETER_1;
}
StartPfn = MI_PFN_ELEMENT (StartPage);
EndPfn = MI_PFN_ELEMENT (EndPage);
ExAcquireFastMutex (&MmDynamicMemoryMutex);
#if DBG
MiDynmemData[0] += 1;
#endif
//
// Decrease all commit limits to reflect the removed memory.
//
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
ASSERT (MmTotalCommitLimit <= MmTotalCommitLimitMaximum);
if ((NumberOfPages + 100 > MmTotalCommitLimit - MmTotalCommittedPages) ||
(MmTotalCommittedPages > MmTotalCommitLimit)) {
#if DBG
MiDynmemData[1] += 1;
#endif
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return STATUS_INSUFFICIENT_RESOURCES;
}
MmTotalCommitLimit -= NumberOfPages;
MmTotalCommitLimitMaximum -= NumberOfPages;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
//
// Check for outstanding promises that cannot be broken.
//
LOCK_PFN (OldIrql);
MaxPages = MI_NONPAGABLE_MEMORY_AVAILABLE() - 100;
if ((SPFN_NUMBER)NumberOfPages > MaxPages) {
#if DBG
MiDynmemData[2] += 1;
#endif
UNLOCK_PFN (OldIrql);
Status = STATUS_INSUFFICIENT_RESOURCES;
goto giveup2;
}
MmResidentAvailablePages -= NumberOfPages;
MmNumberOfPhysicalPages -= NumberOfPages;
//
// The range must be contained in a single entry. It is permissible for
// it to be part of a single entry, but it must not cross multiple entries.
//
Additional = (ULONG)-2;
start = 0;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
if ((StartPage >= Page) && (EndPage <= LastPage)) {
if ((StartPage == Page) && (EndPage == LastPage)) {
Additional = (ULONG)-1;
}
else if ((StartPage == Page) || (EndPage == LastPage)) {
Additional = 0;
}
else {
Additional = 1;
}
break;
}
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
if (Additional == (ULONG)-2) {
#if DBG
MiDynmemData[3] += 1;
#endif
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += NumberOfPages;
UNLOCK_PFN (OldIrql);
Status = STATUS_CONFLICTING_ADDRESSES;
goto giveup2;
}
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
Pfn1->u3.e1.RemovalRequested = 1;
}
//
// The free and zero lists must be pruned now before releasing the PFN
// lock otherwise if another thread allocates the page from these lists,
// the allocation will clear the RemovalRequested flag forever.
//
RemovedPages = MiRemovePhysicalPages (StartPage, EndPage);
if (RemovedPages != NumberOfPages) {
#if DBG
retry:
#endif
Pfn1 = StartPfn;
InterlockedIncrement (&MiDelayPageFaults);
for (i = 0; i < 5; i += 1) {
UNLOCK_PFN (OldIrql);
//
// Attempt to move pages to the standby list. Note that only the
// pages with RemovalRequested set are moved.
//
MiTrimRemovalPagesOnly = TRUE;
MiEmptyAllWorkingSets ();
MiTrimRemovalPagesOnly = FALSE;
MiFlushAllPages ();
KeDelayExecutionThread (KernelMode, FALSE, &MmHalfSecond);
LOCK_PFN (OldIrql);
RemovedPages += MiRemovePhysicalPages (StartPage, EndPage);
if (RemovedPages == NumberOfPages) {
break;
}
//
// RemovedPages doesn't include pages that were freed directly to
// the bad page list via MiDecrementReferenceCount. So use the above
// check purely as an optimization - and walk here when necessary.
//
for ( ; Pfn1 < EndPfn; Pfn1 += 1) {
if (Pfn1->u3.e1.PageLocation != BadPageList) {
break;
}
}
if (Pfn1 == EndPfn) {
RemovedPages = NumberOfPages;
break;
}
}
InterlockedDecrement (&MiDelayPageFaults);
}
if (RemovedPages != NumberOfPages) {
#if DBG
MiDynmemData[4] += 1;
if (MiShowStuckPages != 0) {
RemovedPages = 0;
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
if (Pfn1->u3.e1.PageLocation != BadPageList) {
RemovedPages += 1;
}
}
ASSERT (RemovedPages != 0);
DbgPrint("MmRemovePhysicalMemory : could not get %d of %d pages\n",
RemovedPages, NumberOfPages);
if (MiShowStuckPages & 0x2) {
ULONG PfnsPrinted;
ULONG EnoughShown;
PMMPFN FirstPfn;
PFN_COUNT PfnCount;
PfnCount = 0;
PfnsPrinted = 0;
EnoughShown = 100;
if (MiShowStuckPages & 0x4) {
EnoughShown = (ULONG)-1;
}
DbgPrint("Stuck PFN list: ");
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
if (Pfn1->u3.e1.PageLocation != BadPageList) {
if (PfnCount == 0) {
FirstPfn = Pfn1;
}
PfnCount += 1;
}
else {
if (PfnCount != 0) {
DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
(FirstPfn - MmPfnDatabase) + PfnCount - 1);
PfnsPrinted += 1;
if (PfnsPrinted == EnoughShown) {
break;
}
PfnCount = 0;
}
}
}
if (PfnCount != 0) {
DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
(FirstPfn - MmPfnDatabase) + PfnCount - 1);
}
DbgPrint("\n");
}
if (MiShowStuckPages & 0x8) {
DbgBreakPoint ();
}
if (MiShowStuckPages & 0x10) {
goto retry;
}
}
#endif
UNLOCK_PFN (OldIrql);
Status = STATUS_NO_MEMORY;
goto giveup;
}
#if DBG
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
}
#endif
//
// All the pages in the range have been removed. Update the physical
// memory blocks and other associated housekeeping.
//
if (Additional == 0) {
//
// The range can be split off from an end of an existing chunk so no
// pool growth or shrinkage is required.
//
NewPhysicalMemoryBlock = MmPhysicalMemoryBlock;
OldPhysicalMemoryBlock = NULL;
}
else {
//
// The range cannot be split off from an end of an existing chunk so
// pool growth or shrinkage is required.
//
UNLOCK_PFN (OldIrql);
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + Additional)));
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
i,
' mM');
if (NewPhysicalMemoryBlock == NULL) {
Status = STATUS_INSUFFICIENT_RESOURCES;
#if DBG
MiDynmemData[5] += 1;
#endif
goto giveup;
}
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
RtlZeroMemory (NewPhysicalMemoryBlock, i);
LOCK_PFN (OldIrql);
}
//
// Remove or split the requested range from the existing memory block.
//
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + Additional;
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages - NumberOfPages;
NewRun = &NewPhysicalMemoryBlock->Run[0];
start = 0;
Inserted = FALSE;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;
if (Inserted == FALSE) {
if ((StartPage >= Page) && (EndPage <= LastPage)) {
if ((StartPage == Page) && (EndPage == LastPage)) {
ASSERT (Additional == -1);
start += 1;
continue;
}
else if ((StartPage == Page) || (EndPage == LastPage)) {
ASSERT (Additional == 0);
if (StartPage == Page) {
MmPhysicalMemoryBlock->Run[start].BasePage += NumberOfPages;
}
MmPhysicalMemoryBlock->Run[start].PageCount -= NumberOfPages;
}
else {
ASSERT (Additional == 1);
OriginalLastPage = LastPage;
MmPhysicalMemoryBlock->Run[start].PageCount =
StartPage - MmPhysicalMemoryBlock->Run[start].BasePage;
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
NewRun->BasePage = EndPage;
NewRun->PageCount = OriginalLastPage - EndPage;
NewRun += 1;
start += 1;
continue;
}
Inserted = TRUE;
}
}
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Repoint the MmPhysicalMemoryBlock at the new chunk.
// Free the old block after releasing the PFN lock.
//
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
if (EndPage - 1 == MmHighestPhysicalPage) {
MmHighestPhysicalPage = StartPage - 1;
}
//
// Throw away all the removed pages that are currently enqueued.
//
for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
MiUnlinkPageFromList (Pfn1);
ASSERT (Pfn1->u1.Flink == 0);
ASSERT (Pfn1->u2.Blink == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
Pfn1->PteAddress = PFN_REMOVED;
Pfn1->u3.e2.ShortFlags = 0;
Pfn1->OriginalPte.u.Long = ZeroKernelPte.u.Long;
Pfn1->PteFrame = 0;
}
//
// Now that the removed pages have been discarded, eliminate the PFN
// entries that mapped them. Straddling entries left over from an
// adjacent earlier removal are not collapsed at this point.
//
//
PagesReleased = 0;
if (PfnDatabaseIsPhysical == FALSE) {
VirtualAddress = (PVOID)ROUND_TO_PAGES(MI_PFN_ELEMENT(StartPage));
PointerPte = MiGetPteAddress (VirtualAddress);
EndPte = MiGetPteAddress (PAGE_ALIGN(MI_PFN_ELEMENT(EndPage)));
while (PointerPte < EndPte) {
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u2.ShareCount == 1);
ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
Pfn1->u2.ShareCount = 0;
MI_SET_PFN_DELETED (Pfn1);
#if DBG
Pfn1->u3.e1.PageLocation = StandbyPageList;
#endif //DBG
MiDecrementReferenceCount (PageFrameIndex);
KeFlushSingleTb (VirtualAddress,
TRUE,
TRUE,
(PHARDWARE_PTE)PointerPte,
ZeroKernelPte.u.Flush);
PagesReleased += 1;
PointerPte += 1;
VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE);
}
MmResidentAvailablePages += PagesReleased;
}
#if DBG
MiDynmemData[6] += 1;
#endif
UNLOCK_PFN (OldIrql);
if (PagesReleased != 0) {
MiReturnCommitment (PagesReleased);
}
ExReleaseFastMutex (&MmDynamicMemoryMutex);
if (OldPhysicalMemoryBlock != NULL) {
ExFreePool (OldPhysicalMemoryBlock);
}
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
return STATUS_SUCCESS;
giveup:
//
// All the pages in the range were not obtained. Back everything out.
//
PageFrameIndex = StartPage;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
LOCK_PFN (OldIrql);
while (PageFrameIndex < EndPage) {
ASSERT (Pfn1->u3.e1.RemovalRequested == 1);
Pfn1->u3.e1.RemovalRequested = 0;
if ((Pfn1->u3.e1.PageLocation == BadPageList) &&
(Pfn1->u3.e1.ParityError == 0)) {
MiUnlinkPageFromList (Pfn1);
MiInsertPageInList (MmPageLocationList[FreePageList],
PageFrameIndex);
}
Pfn1 += 1;
PageFrameIndex += 1;
}
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += NumberOfPages;
UNLOCK_PFN (OldIrql);
giveup2:
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
MmTotalCommitLimit += NumberOfPages;
MmTotalCommitLimitMaximum += NumberOfPages;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return Status;
}
BOOLEAN
MmCreateProcessAddressSpace (
IN ULONG MinimumWorkingSetSize,
IN PEPROCESS NewProcess,
OUT PULONG_PTR DirectoryTableBase
)
/*++
Routine Description:
This routine creates an address space which maps the system
portion and contains a hyper space entry.
Arguments:
MinimumWorkingSetSize - Supplies the minimum working set size for
this address space. This value is only used
to ensure that ample physical pages exist
to create this process.
NewProcess - Supplies a pointer to the process object being created.
DirectoryTableBase - Returns the value of the newly created
address space's Page Directory (PD) page and
hyper space page.
Return Value:
Returns TRUE if an address space was successfully created, FALSE
if ample physical pages do not exist.
Environment:
Kernel mode. APCs Disabled.
--*/
{
LOGICAL FlushTbNeeded;
PFN_NUMBER PageDirectoryIndex;
PFN_NUMBER HyperSpaceIndex;
PFN_NUMBER PageContainingWorkingSet;
PFN_NUMBER VadBitMapPage;
MMPTE TempPte;
MMPTE TempPte2;
PEPROCESS CurrentProcess;
KIRQL OldIrql;
PMMPFN Pfn1;
ULONG Color;
PMMPTE PointerPte;
ULONG PdeOffset;
PMMPTE MappingPte;
PMMPTE PointerFillPte;
PMMPTE CurrentAddressSpacePde;
//
// Charge commitment for the page directory pages, working set page table
// page, and working set list. If Vad bitmap lookups are enabled, then
// charge for a page or two for that as well.
//
if (MiChargeCommitment (MM_PROCESS_COMMIT_CHARGE, NULL) == FALSE) {
return FALSE;
}
FlushTbNeeded = FALSE;
CurrentProcess = PsGetCurrentProcess ();
NewProcess->NextPageColor = (USHORT) (RtlRandom (&MmProcessColorSeed));
KeInitializeSpinLock (&NewProcess->HyperSpaceLock);
//
// Get the PFN lock to get physical pages.
//
LOCK_PFN (OldIrql);
//
// Check to make sure the physical pages are available.
//
if (MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MinimumWorkingSetSize){
UNLOCK_PFN (OldIrql);
MiReturnCommitment (MM_PROCESS_COMMIT_CHARGE);
//
// Indicate no directory base was allocated.
//
return FALSE;
}
MM_TRACK_COMMIT (MM_DBG_COMMIT_PROCESS_CREATE, MM_PROCESS_COMMIT_CHARGE);
MI_DECREMENT_RESIDENT_AVAILABLE (MinimumWorkingSetSize,
MM_RESAVAIL_ALLOCATE_CREATE_PROCESS);
//
// Allocate a page directory page.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_PTE_PROCESS (PDE_BASE,
&CurrentProcess->NextPageColor);
PageDirectoryIndex = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (PageDirectoryIndex);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Allocate the hyper space page table page.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_PTE_PROCESS (MiGetPdeAddress(HYPER_SPACE),
&CurrentProcess->NextPageColor);
HyperSpaceIndex = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (HyperSpaceIndex);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Remove page(s) for the VAD bitmap.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_VA_PROCESS (MmWorkingSetList,
&CurrentProcess->NextPageColor);
VadBitMapPage = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (VadBitMapPage);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Remove a page for the working set list.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_VA_PROCESS (MmWorkingSetList,
&CurrentProcess->NextPageColor);
PageContainingWorkingSet = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (PageContainingWorkingSet);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
UNLOCK_PFN (OldIrql);
if (FlushTbNeeded == TRUE) {
MI_FLUSH_TB_FOR_CACHED_ATTRIBUTE ();
}
ASSERT (NewProcess->AddressSpaceInitialized == 0);
PS_SET_BITS (&NewProcess->Flags, PS_PROCESS_FLAGS_ADDRESS_SPACE1);
ASSERT (NewProcess->AddressSpaceInitialized == 1);
NewProcess->Vm.MinimumWorkingSetSize = MinimumWorkingSetSize;
NewProcess->WorkingSetPage = PageContainingWorkingSet;
INITIALIZE_DIRECTORY_TABLE_BASE (&DirectoryTableBase[0], PageDirectoryIndex);
INITIALIZE_DIRECTORY_TABLE_BASE (&DirectoryTableBase[1], HyperSpaceIndex);
//
// Initialize the page reserved for hyper space.
//
TempPte = ValidPdePde;
MI_SET_GLOBAL_STATE (TempPte, 0);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
HyperSpaceIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
PointerPte = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
PointerPte = MiMapPageInHyperSpace (CurrentProcess, HyperSpaceIndex, &OldIrql);
}
TempPte.u.Hard.PageFrameNumber = VadBitMapPage;
PointerPte[MiGetPteOffset(VAD_BITMAP_SPACE)] = TempPte;
TempPte.u.Hard.PageFrameNumber = PageContainingWorkingSet;
PointerPte[MiGetPteOffset(MmWorkingSetList)] = TempPte;
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, PointerPte, OldIrql);
}
//
// Set the PTE address in the PFN for the page directory page.
//
Pfn1 = MI_PFN_ELEMENT (PageDirectoryIndex);
Pfn1->PteAddress = (PMMPTE)PDE_BASE;
TempPte = ValidPdePde;
TempPte.u.Hard.PageFrameNumber = HyperSpaceIndex;
MI_SET_GLOBAL_STATE (TempPte, 0);
//
// Add the new process to our internal list prior to filling any
// system PDEs so if a system PDE changes (large page map or unmap)
// it can mark this process for a subsequent update.
//
ASSERT (NewProcess->Pcb.DirectoryTableBase[0] == 0);
LOCK_EXPANSION (OldIrql);
InsertTailList (&MmProcessList, &NewProcess->MmProcessLinks);
UNLOCK_EXPANSION (OldIrql);
//
// Map the page directory page in hyperspace.
//
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
PageDirectoryIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
PointerPte = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
PointerPte = MiMapPageInHyperSpace (CurrentProcess, PageDirectoryIndex, &OldIrql);
}
PdeOffset = MiGetPdeOffset (MmSystemRangeStart);
PointerFillPte = &PointerPte[PdeOffset];
CurrentAddressSpacePde = MiGetPdeAddress (MmSystemRangeStart);
RtlCopyMemory (PointerFillPte,
CurrentAddressSpacePde,
PAGE_SIZE - PdeOffset * sizeof (MMPTE));
//
// Map the working set page table page.
//
PdeOffset = MiGetPdeOffset (HYPER_SPACE);
PointerPte[PdeOffset] = TempPte;
//
// Zero the remaining page directory range used to map the working
// set list and its hash.
//
PdeOffset += 1;
ASSERT (MiGetPdeOffset (MmHyperSpaceEnd) >= PdeOffset);
MiZeroMemoryPte (&PointerPte[PdeOffset],
(MiGetPdeOffset (MmHyperSpaceEnd) - PdeOffset + 1));
//
// Recursively map the page directory page so it points to itself.
//
TempPte.u.Hard.PageFrameNumber = PageDirectoryIndex;
PointerPte[MiGetPdeOffset(PTE_BASE)] = TempPte;
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, PointerPte, OldIrql);
}
InterlockedExchangeAddSizeT (&MmProcessCommit, MM_PROCESS_COMMIT_CHARGE);
//
// Up the session space reference count.
//
MiSessionAddProcess (NewProcess);
return TRUE;
}
VOID
NTAPI
MmFreeSpecialPool(PVOID P)
{
PMMPTE PointerPte;
PPOOL_HEADER Header;
BOOLEAN Overruns = FALSE;
KIRQL Irql = KeGetCurrentIrql();
POOL_TYPE PoolType;
ULONG BytesRequested, BytesReal = 0;
ULONG PtrOffset;
PUCHAR b;
PMI_FREED_SPECIAL_POOL FreedHeader;
LARGE_INTEGER TickCount;
PMMPFN Pfn;
DPRINT1("MmFreeSpecialPool(%p)\n", P);
/* Get the PTE */
PointerPte = MiAddressToPte(P);
/* Check if it's valid */
if (PointerPte->u.Hard.Valid == 0)
{
/* Bugcheck if it has NOACCESS or 0 set as protection */
if (PointerPte->u.Soft.Protection == MM_NOACCESS ||
!PointerPte->u.Soft.Protection)
{
KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, (ULONG_PTR)PointerPte, 0, 0x20);
}
}
/* Determine if it's a underruns or overruns pool pointer */
PtrOffset = (ULONG)((ULONG_PTR)P & (PAGE_SIZE - 1));
if (PtrOffset)
{
/* Pool catches overruns */
Header = PAGE_ALIGN(P);
Overruns = TRUE;
}
else
{
/* Pool catches underruns */
Header = (PPOOL_HEADER)((PUCHAR)PAGE_ALIGN(P) + PAGE_SIZE - sizeof(POOL_HEADER));
}
/* Check if it's non paged pool */
if ((Header->Ulong1 & SPECIAL_POOL_PAGED) == 0)
{
/* Non-paged allocation, ensure that IRQ is not higher that DISPATCH */
ASSERT((PointerPte + 1)->u.Soft.PageFileHigh == SPECIAL_POOL_NONPAGED_PTE);
if (Irql > DISPATCH_LEVEL)
{
KeBugCheckEx(BAD_POOL_HEADER, Irql, (ULONG_PTR)P, 0, 0x31);
}
PoolType = NonPagedPool;
}
else
{
/* Paged allocation, ensure */
ASSERT((PointerPte + 1)->u.Soft.PageFileHigh == SPECIAL_POOL_PAGED_PTE);
if (Irql > DISPATCH_LEVEL)
{
KeBugCheckEx(BAD_POOL_HEADER, Irql, (ULONG_PTR)P, 1, 0x31);
}
PoolType = PagedPool;
}
/* Get amount of bytes user requested to be allocated by clearing out the paged mask */
BytesRequested = (Header->Ulong1 & ~SPECIAL_POOL_PAGED) & 0xFFFF;
/* Check memory before the allocated user buffer in case of overruns detection */
if (Overruns)
{
/* Calculate the real placement of the buffer */
BytesReal = PAGE_SIZE - PtrOffset;
/* If they mismatch, it's unrecoverable */
if (BytesRequested > BytesReal)
{
KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, BytesRequested, BytesReal, 0x21);
}
if (BytesRequested + sizeof(POOL_HEADER) < BytesReal)
{
KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, BytesRequested, BytesReal, 0x22);
}
/* Actually check the memory pattern */
for (b = (PUCHAR)(Header + 1); b < (PUCHAR)P; b++)
{
if (Header->BlockSize != b[0])
{
/* Bytes mismatch */
KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, (ULONG_PTR)b, Header->BlockSize, 0x23);
}
}
}
/* Check the memory pattern after the user buffer */
MiSpecialPoolCheckPattern(P, Header);
/* Fill the freed header */
KeQueryTickCount(&TickCount);
FreedHeader = (PMI_FREED_SPECIAL_POOL)PAGE_ALIGN(P);
FreedHeader->Signature = 0x98764321;
FreedHeader->TickCount = TickCount.LowPart;
FreedHeader->NumberOfBytesRequested = BytesRequested;
FreedHeader->Pagable = PoolType;
FreedHeader->VirtualAddress = P;
FreedHeader->Thread = PsGetCurrentThread();
/* TODO: Fill StackPointer and StackBytes */
FreedHeader->StackPointer = NULL;
FreedHeader->StackBytes = 0;
if (PoolType == NonPagedPool)
{
/* Non pagable. Get PFN element corresponding to the PTE */
Pfn = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
/* Lock PFN database */
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Delete this PFN */
MI_SET_PFN_DELETED(Pfn);
/* Decrement share count of this PFN */
MiDecrementShareCount(Pfn, PointerPte->u.Hard.PageFrameNumber);
/* Flush the TLB */
//FIXME: Use KeFlushSingleTb() instead
KeFlushEntireTb(TRUE, TRUE);
}
else
{
/* Pagable. Delete that virtual address */
MiDeleteSystemPageableVm(PointerPte, 1, 0, NULL);
/* Lock PFN database */
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
}
/* Mark next PTE as invalid */
PointerPte[1].u.Long = 0; //|= 8000;
/* Make sure that the last entry is really the last one */
ASSERT(MiSpecialPoolLastPte->u.List.NextEntry == MM_EMPTY_PTE_LIST);
/* Update the current last PTE next pointer */
MiSpecialPoolLastPte->u.List.NextEntry = PointerPte - MmSystemPteBase;
/* PointerPte becomes the new last PTE */
PointerPte->u.List.NextEntry = MM_EMPTY_PTE_LIST;
MiSpecialPoolLastPte = PointerPte;
/* Release the PFN database lock */
KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql);
}
VOID
MmZeroPageThread (
VOID
)
/*++
Routine Description:
Implements the NT zeroing page thread. This thread runs
at priority zero and removes a page from the free list,
zeroes it, and places it on the zeroed page list.
Arguments:
StartContext - not used.
Return Value:
None.
Environment:
Kernel mode.
--*/
{
PVOID EndVa;
KIRQL OldIrql;
ULONG PageFrame;
PMMPFN Pfn1;
PVOID StartVa;
PKTHREAD Thread;
PVOID ZeroBase;
ULONG i;
//
// Before this becomes the zero page thread, free the kernel
// initialization code.
//
MiFindInitializationCode (&StartVa, &EndVa);
if (StartVa != NULL) {
MiFreeInitializationCode (StartVa, EndVa);
}
//
// The following code sets the current thread's base priority to zero
// and then sets its current priority to zero. This ensures that the
// thread always runs at a priority of zero.
//
Thread = KeGetCurrentThread();
Thread->BasePriority = 0;
KeSetPriorityThread (Thread, 0);
//
// Loop forever zeroing pages.
//
do {
//
// Wait until there are at least MmZeroPageMinimum pages
// on the free list.
//
KeWaitForSingleObject (&MmZeroingPageEvent,
WrFreePage,
KernelMode,
FALSE,
(PLARGE_INTEGER)NULL);
LOCK_PFN_WITH_TRY (OldIrql);
do {
if ((volatile)MmFreePageListHead.Total == 0) {
//
// No pages on the free list at this time, wait for
// some more.
//
MmZeroingPageThreadActive = FALSE;
UNLOCK_PFN (OldIrql);
break;
} else {
#if MM_MAXIMUM_NUMBER_OF_COLORS > 1
for (i = 0; i < MM_MAXIMUM_NUMBER_OF_COLORS; i++) {
PageFrame = MmFreePagesByPrimaryColor[FreePageList][i].Flink;
if (PageFrame != MM_EMPTY_LIST) {
break;
}
}
#else //MM_MAXIMUM_NUMBER_OF_COLORS > 1
PageFrame = MmFreePageListHead.Flink;
#endif //MM_MAXIMUM_NUMBER_OF_COLORS > 1
Pfn1 = MI_PFN_ELEMENT(PageFrame);
ASSERT (PageFrame != MM_EMPTY_LIST);
Pfn1 = MI_PFN_ELEMENT(PageFrame);
MiRemoveAnyPage (MI_GET_SECONDARY_COLOR (PageFrame, Pfn1));
//
// Zero the page using the last color used to map the page.
//
#if defined(_X86_)
ZeroBase = MiMapPageToZeroInHyperSpace (PageFrame);
UNLOCK_PFN (OldIrql);
RtlZeroMemory (ZeroBase, PAGE_SIZE);
#elif defined(_PPC_)
UNLOCK_PFN (OldIrql);
KeZeroPage(PageFrame);
#else
ZeroBase = (PVOID)(Pfn1->u3.e1.PageColor << PAGE_SHIFT);
UNLOCK_PFN (OldIrql);
HalZeroPage(ZeroBase, ZeroBase, PageFrame);
#endif //X86
LOCK_PFN_WITH_TRY (OldIrql);
MiInsertPageInList (MmPageLocationList[ZeroedPageList],
PageFrame);
}
} while(TRUE);
} while (TRUE);
}
VOID
FASTCALL
MiDecrementReferenceCount (
IN PMMPFN Pfn1,
IN PFN_NUMBER PageFrameIndex
)
/*++
Routine Description:
This routine decrements the reference count for the specified page.
If the reference count becomes zero, the page is placed on the
appropriate list (free, modified, standby or bad). If the page
is placed on the free or standby list, the number of available
pages is incremented and if it transitions from zero to one, the
available page event is set.
Arguments:
Pfn1 - Supplies the PFN database entry to decrement.
PageFrameIndex - Supplies the physical page number of which to
decrement the reference count.
Return Value:
None.
Environment:
Must be holding the PFN database lock with APCs disabled.
--*/
{
ULONG FreeBit;
MM_PFN_LOCK_ASSERT();
ASSERT (MI_IS_PFN (PageFrameIndex));
ASSERT (Pfn1 == MI_PFN_ELEMENT (PageFrameIndex));
ASSERT (Pfn1->u3.e2.ReferenceCount != 0);
InterlockedDecrementPfn ((PSHORT)&Pfn1->u3.e2.ReferenceCount);
if (Pfn1->u3.e2.ReferenceCount != 0) {
//
// The reference count is not zero, return.
//
return;
}
//
// The reference count is now zero, put the page on some list.
//
if (Pfn1->u2.ShareCount != 0) {
KeBugCheckEx (PFN_LIST_CORRUPT,
7,
PageFrameIndex,
Pfn1->u2.ShareCount,
0);
return;
}
ASSERT (Pfn1->u3.e1.PageLocation != ActiveAndValid);
if (MI_IS_PFN_DELETED (Pfn1)) {
//
// There is no referenced PTE for this page, delete the page
// file space (if any), and place the page on the free list.
//
if (Pfn1->OriginalPte.u.Soft.Prototype == 0) {
FreeBit = GET_PAGING_FILE_OFFSET (Pfn1->OriginalPte);
if ((FreeBit != 0) && (FreeBit != MI_PTE_LOOKUP_NEEDED)) {
MiReleaseConfirmedPageFileSpace (Pfn1->OriginalPte);
}
}
MiInsertPageInFreeList (PageFrameIndex);
return;
}
//
// Place the page on the modified or standby list depending
// on the state of the modify bit in the PFN element.
//
if (Pfn1->u3.e1.Modified == 1) {
MiInsertPageInList (&MmModifiedPageListHead, PageFrameIndex);
}
else {
if (Pfn1->u3.e1.RemovalRequested == 1) {
//
// The page may still be marked as on the modified list if the
// current thread is the modified writer completing the write.
// Mark it as standby so restoration of the transition PTE
// doesn't flag this as illegal.
//
Pfn1->u3.e1.PageLocation = StandbyPageList;
MiRestoreTransitionPte (Pfn1);
MiInsertPageInList (&MmBadPageListHead, PageFrameIndex);
return;
}
if (!MmFrontOfList) {
MiInsertPageInList (&MmStandbyPageListHead, PageFrameIndex);
}
else {
MiInsertStandbyListAtFront (PageFrameIndex);
}
}
return;
}
VOID
MiCheckPfn (
)
/*++
Routine Description:
This routine checks each physical page in the PFN database to ensure
it is in the proper state.
Arguments:
None.
Return Value:
None.
Environment:
Kernel mode, APCs disabled.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER Link, Previous;
ULONG i;
PMMPTE PointerPte;
KIRQL PreviousIrql;
KIRQL OldIrql;
USHORT ValidCheck[4];
USHORT ValidPage[4];
PMMPFN PfnX;
ValidCheck[0] = ValidCheck[1] = ValidCheck[2] = ValidCheck[3] = 0;
ValidPage[0] = ValidPage[1] = ValidPage[2] = ValidPage[3] = 0;
if (CheckPfnBitMap == NULL) {
MiCreateBitMap ( &CheckPfnBitMap, MmNumberOfPhysicalPages, NonPagedPool);
}
RtlClearAllBits (CheckPfnBitMap);
//
// Walk free list.
//
KeRaiseIrql (APC_LEVEL, &PreviousIrql);
LOCK_PFN (OldIrql);
Previous = MM_EMPTY_LIST;
Link = MmFreePageListHead.Flink;
for (i=0; i < MmFreePageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on free list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != FreePageList) {
DbgPrint("page location not freelist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on free list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk zeroed list.
//
Previous = MM_EMPTY_LIST;
Link = MmZeroedPageListHead.Flink;
for (i=0; i < MmZeroedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on zero list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ZeroedPageList) {
DbgPrint("page location not zerolist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on zero list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Bad list.
//
Previous = MM_EMPTY_LIST;
Link = MmBadPageListHead.Flink;
for (i=0; i < MmBadPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Bad list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != BadPageList) {
DbgPrint("page location not Badlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Bad list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Standby list.
//
Previous = MM_EMPTY_LIST;
Link = MmStandbyPageListHead.Flink;
for (i=0; i < MmStandbyPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Standby list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != StandbyPageList) {
DbgPrint("page location not Standbylist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Standby list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in standby list\n");
MiFormatPfn(Pfn1);
} else {
OldIrql = 99;
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame,
&OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Modified list.
//
Previous = MM_EMPTY_LIST;
Link = MmModifiedPageListHead.Flink;
for (i=0; i < MmModifiedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Modified list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ModifiedPageList) {
DbgPrint("page location not Modifiedlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Modified list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in modified list\n");
MiFormatPfn(Pfn1);
} else {
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// All non active pages have been scanned. Locate the
// active pages and make sure they are consistent.
//
//
// set bit zero as page zero is reserved for now
//
RtlSetBits (CheckPfnBitMap, 0L, 1L);
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
while (Link != 0xFFFFFFFF) {
Pfn1 = MI_PFN_ELEMENT (Link);
//
// Make sure the PTE address is okay
//
if ((Pfn1->PteAddress >= (PMMPTE)HYPER_SPACE)
&& (Pfn1->u3.e1.PrototypePte == 0)) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
if (Pfn1->PteAddress < (PMMPTE)PTE_BASE) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
#if defined(_IA64_)
//
// ignore PTEs mapped to IA64 kernel BAT.
//
if (MI_IS_PHYSICAL_ADDRESS(MiGetVirtualAddressMappedByPte(Pfn1->PteAddress))) {
goto NoCheck;
}
#endif // _IA64_
#ifdef _ALPHA_
//
// ignore ptes mapped to ALPHA's 32-bit superpage.
//
if ((Pfn1->PteAddress > (PMMPTE)(ULONG_PTR)0xc0100000) &&
(Pfn1->PteAddress < (PMMPTE)(ULONG_PTR)0xc0180000)) {
goto NoCheck;
}
#endif //ALPHA
//
// Check to make sure the referenced PTE is for this page.
//
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Hard.Valid == 0) {
//
// if the page is a page table page it could be out of
// the working set yet a transition page is keeping it
// around in memory (ups the share count).
//
if ((Pfn1->PteAddress < (PMMPTE)PDE_BASE) ||
(Pfn1->PteAddress > (PMMPTE)PDE_TOP)) {
DbgPrint("Pte not valid for page on active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
}
if (Pfn1->u3.e2.ReferenceCount != 1) {
DbgPrint("refcount not 1\n");
MiFormatPfn(Pfn1);
}
//
// Check to make sure the PTE count for the frame is okay.
//
if (Pfn1->u3.e1.PrototypePte == 1) {
PfnX = MI_PFN_ELEMENT(Pfn1->PteFrame);
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
ValidPage[i] = (USHORT)Pfn1->PteFrame;
}
if (ValidPage[i] == (USHORT)Pfn1->PteFrame) {
ValidCheck[i] += 1;
break;
}
}
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
#if defined(_ALPHA_) || defined(_IA64_)
NoCheck:
#endif
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
}
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
break;
}
PfnX = MI_PFN_ELEMENT(ValidPage[i]);
}
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
VOID
FASTCALL
MiDecrementShareCount (
IN PMMPFN Pfn1,
IN PFN_NUMBER PageFrameIndex
)
/*++
Routine Description:
This routine decrements the share count within the PFN element
for the specified physical page. If the share count becomes
zero the corresponding PTE is converted to the transition state
and the reference count is decremented and the ValidPte count
of the PTEframe is decremented.
Arguments:
Pfn1 - Supplies the PFN database entry to decrement.
PageFrameIndex - Supplies the physical page number of which to decrement
the share count.
Return Value:
None.
Environment:
Must be holding the PFN database lock with APCs disabled.
--*/
{
ULONG FreeBit;
MMPTE TempPte;
PMMPTE PointerPte;
PEPROCESS Process;
ASSERT (PageFrameIndex > 0);
ASSERT (MI_IS_PFN (PageFrameIndex));
ASSERT (Pfn1 == MI_PFN_ELEMENT (PageFrameIndex));
if (Pfn1->u3.e1.PageLocation != ActiveAndValid &&
Pfn1->u3.e1.PageLocation != StandbyPageList) {
KeBugCheckEx (PFN_LIST_CORRUPT,
0x99,
PageFrameIndex,
Pfn1->u3.e1.PageLocation,
0);
}
Pfn1->u2.ShareCount -= 1;
ASSERT (Pfn1->u2.ShareCount < 0xF000000);
if (Pfn1->u2.ShareCount == 0) {
if (PERFINFO_IS_GROUP_ON(PERF_MEMORY)) {
PERFINFO_PFN_INFORMATION PerfInfoPfn;
PerfInfoPfn.PageFrameIndex = PageFrameIndex;
PerfInfoLogBytes(PERFINFO_LOG_TYPE_ZEROSHARECOUNT,
&PerfInfoPfn,
sizeof(PerfInfoPfn));
}
//
// The share count is now zero, decrement the reference count
// for the PFN element and turn the referenced PTE into
// the transition state if it refers to a prototype PTE.
// PTEs which are not prototype PTEs do not need to be placed
// into transition as they are placed in transition when
// they are removed from the working set (working set free routine).
//
//
// If the PTE referenced by this PFN element is actually
// a prototype PTE, it must be mapped into hyperspace and
// then operated on.
//
if (Pfn1->u3.e1.PrototypePte == 1) {
if (MiIsProtoAddressValid (Pfn1->PteAddress)) {
Process = NULL;
PointerPte = Pfn1->PteAddress;
}
else {
//
// The address is not valid in this process, map it into
// hyperspace so it can be operated upon.
//
Process = PsGetCurrentProcess ();
PointerPte = (PMMPTE) MiMapPageInHyperSpaceAtDpc(Process, Pfn1->u4.PteFrame);
PointerPte = (PMMPTE)((PCHAR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
TempPte = *PointerPte;
MI_MAKE_VALID_PTE_TRANSITION (TempPte,
Pfn1->OriginalPte.u.Soft.Protection);
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
if (Process != NULL) {
MiUnmapPageInHyperSpaceFromDpc (Process, PointerPte);
}
//
// There is no need to flush the translation buffer at this
// time as we only invalidated a prototype PTE.
//
}
//
// Change the page location to inactive (from active and valid).
//
Pfn1->u3.e1.PageLocation = TransitionPage;
//
// Decrement the reference count as the share count is now zero.
//
MM_PFN_LOCK_ASSERT();
ASSERT (Pfn1->u3.e2.ReferenceCount != 0);
if (Pfn1->u3.e2.ReferenceCount == 1) {
if (MI_IS_PFN_DELETED (Pfn1)) {
Pfn1->u3.e2.ReferenceCount = 0;
//
// There is no referenced PTE for this page, delete the page
// file space (if any), and place the page on the free list.
//
ASSERT (Pfn1->OriginalPte.u.Soft.Prototype == 0);
FreeBit = GET_PAGING_FILE_OFFSET (Pfn1->OriginalPte);
if ((FreeBit != 0) && (FreeBit != MI_PTE_LOOKUP_NEEDED)) {
MiReleaseConfirmedPageFileSpace (Pfn1->OriginalPte);
}
//
// Temporarily mark the frame as active and valid so that
// MiIdentifyPfn knows it is safe to walk back through the
// containing frames for a more accurate identification.
// Note the page will be immediately re-marked as it is
// inserted into the freelist.
//
Pfn1->u3.e1.PageLocation = ActiveAndValid;
MiInsertPageInFreeList (PageFrameIndex);
}
else {
MiDecrementReferenceCount (Pfn1, PageFrameIndex);
}
}
else {
InterlockedDecrementPfn ((PSHORT)&Pfn1->u3.e2.ReferenceCount);
}
}
return;
}
VOID
MiSetDirtyBit (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte,
IN ULONG PfnHeld
)
/*++
Routine Description:
This routine sets dirty in the specified PTE and the modify bit in the
correpsonding PFN element. If any page file space is allocated, it
is deallocated.
Arguments:
FaultingAddress - Supplies the faulting address.
PointerPte - Supplies a pointer to the corresponding valid PTE.
PfnHeld - Supplies TRUE if the PFN mutex is already held.
Return Value:
None.
Environment:
Kernel mode, APC's disabled, Working set mutex held.
--*/
{
MMPTE TempPte;
ULONG PageFrameIndex;
PMMPFN Pfn1;
KIRQL OldIrql;
//
// The page is NOT copy on write, update the PTE setting both the
// dirty bit and the accessed bit. Note, that as this PTE is in
// the TB, the TB must be flushed.
//
PageFrameIndex = PointerPte->u.Hard.PageFrameNumber;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
TempPte = *PointerPte;
TempPte.u.Hard.Dirty = MM_PTE_DIRTY;
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
*PointerPte = TempPte;
//
// Check state of PFN mutex and if not held, don't update PFN database.
//
if (PfnHeld) {
//
// Set the modified field in the PFN database, also, if the phyiscal
// page is currently in a paging file, free up the page file space
// as the contents are now worthless.
//
if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
(Pfn1->u3.e1.WriteInProgress == 0)) {
//
// This page is in page file format, deallocate the page file space.
//
MiReleasePageFileSpace (Pfn1->OriginalPte);
//
// Change original PTE to indicate no page file space is reserved,
// otherwise the space will be deallocated when the PTE is
// deleted.
//
Pfn1->OriginalPte.u.Soft.PageFileHigh = 0;
}
Pfn1->u3.e1.Modified = 1;
}
//
// The TB entry must be flushed as the valid PTE with the dirty bit clear
// has been fetched into the TB. If it isn't flushed, another fault
// is generated as the dirty bit is not set in the cached TB entry.
//
KeFillEntryTb ((PHARDWARE_PTE)PointerPte, FaultingAddress, TRUE);
return;
}
PVOID
MmDbgTranslatePhysicalAddress (
IN PHYSICAL_ADDRESS PhysicalAddress
)
/*++
Routine Description:
MIPS implementation specific:
This routine maps the specified physical address and returns
the virtual address which maps the physical address.
The next call to MmDbgTranslatePhyiscalAddress removes the
previous phyiscal address translation, hence on a single
physical address can be examined at a time (can't cross page
boundaries).
Arguments:
PhysicalAddress - Supplies the phyiscal address to map and translate.
Return Value:
The virtual address which corresponds to the phyiscal address.
NULL if the physical address was bogus.
Environment:
Kernel mode IRQL at DISPATCH_LEVEL or greater.
--*/
{
PVOID BaseAddress;
PMMPTE BasePte;
PMMPFN Pfn1;
ULONG Page;
BasePte = MmDebugPte + (MM_NUMBER_OF_COLORS - 1);
BasePte = (PMMPTE)((ULONG)BasePte & ~(MM_COLOR_MASK << PTE_SHIFT));
Page = (ULONG)(PhysicalAddress.QuadPart >> PAGE_SHIFT);
if ((Page > (LONGLONG)MmHighestPhysicalPage) ||
(Page < (LONGLONG)MmLowestPhysicalPage)) {
return NULL;
}
Pfn1 = MI_PFN_ELEMENT (Page);
if (!MmIsAddressValid (Pfn1)) {
return NULL;
}
BasePte = BasePte + Pfn1->u3.e1.PageColor;
BaseAddress = MiGetVirtualAddressMappedByPte (BasePte);
KiFlushSingleTb (TRUE, BaseAddress);
*BasePte = ValidKernelPte;
BasePte->u.Hard.PageFrameNumber = Page;
return (PVOID)((ULONG)BaseAddress + BYTE_OFFSET(PhysicalAddress.LowPart));
}
