NTSTATUS
MiCcPutPagesInTransition (
IN PMI_READ_INFO MiReadInfo
)
/*++
Routine Description:
This routine allocates physical memory for the specified read-list and
puts all the pages in transition (so collided faults from other threads
for these same pages remain coherent). I/O for any pages not already
resident are issued here. The caller must wait for their completion.
Arguments:
MiReadInfo - Supplies a pointer to the read-list.
Return Value:
STATUS_SUCCESS - all the pages were already resident, reference counts
have been applied and no I/O needs to be waited for.
STATUS_ISSUE_PAGING_IO - the I/O has been issued and the caller must wait.
Various other failure status values indicate the operation failed.
Environment:
Kernel mode. PASSIVE_LEVEL.
--*/
{
NTSTATUS status;
PMMPTE LocalPrototypePte;
PVOID StartingVa;
PFN_NUMBER MdlPages;
KIRQL OldIrql;
MMPTE PteContents;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER ResidentAvailableCharge;
PPFN_NUMBER IoPage;
PPFN_NUMBER ApiPage;
PPFN_NUMBER Page;
PPFN_NUMBER DestinationPage;
ULONG PageColor;
PMMPTE PointerPte;
PMMPTE *ProtoPteArray;
PMMPTE *EndProtoPteArray;
PFN_NUMBER DummyPage;
PMDL Mdl;
PMDL FreeMdl;
PMMPFN PfnProto;
PMMPFN Pfn1;
PMMPFN DummyPfn1;
ULONG i;
PFN_NUMBER DummyTrim;
ULONG NumberOfPagesNeedingIo;
MMPTE TempPte;
PMMPTE PointerPde;
PEPROCESS CurrentProcess;
PMMINPAGE_SUPPORT InPageSupport;
PKPRCB Prcb;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
MiReadInfo->DummyPagePfn = NULL;
FreeMdl = NULL;
CurrentProcess = PsGetCurrentProcess();
PfnProto = NULL;
PointerPde = NULL;
InPageSupport = MiReadInfo->InPageSupport;
Mdl = MI_EXTRACT_PREFETCH_MDL (InPageSupport);
ASSERT (Mdl == MiReadInfo->IoMdl);
IoPage = (PPFN_NUMBER)(Mdl + 1);
ApiPage = (PPFN_NUMBER)(MiReadInfo->ApiMdl + 1);
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
MdlPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES (StartingVa,
Mdl->ByteCount);
if (MdlPages + 1 > MAXUSHORT) {
//
// The PFN ReferenceCount for the dummy page could wrap, refuse the
// request.
//
return STATUS_INSUFFICIENT_RESOURCES;
}
NumberOfPagesNeedingIo = 0;
ProtoPteArray = (PMMPTE *)InPageSupport->BasePte;
EndProtoPteArray = ProtoPteArray + MdlPages;
ASSERT (*ProtoPteArray != NULL);
LOCK_PFN (OldIrql);
//
// Ensure sufficient pages exist for the transfer plus the dummy page.
//
if (((SPFN_NUMBER)MdlPages > (SPFN_NUMBER)(MmAvailablePages - MM_HIGH_LIMIT)) ||
(MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MdlPages)) {
UNLOCK_PFN (OldIrql);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// Charge resident available immediately as the PFN lock may get released
// and reacquired below before all the pages have been locked down.
// Note the dummy page is immediately charged separately.
//
MI_DECREMENT_RESIDENT_AVAILABLE (MdlPages, MM_RESAVAIL_ALLOCATE_BUILDMDL);
ResidentAvailableCharge = MdlPages;
//
// Allocate a dummy page to map discarded pages that aren't skipped.
//
DummyPage = MiRemoveAnyPage (0);
Pfn1 = MI_PFN_ELEMENT (DummyPage);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MiInitializePfnForOtherProcess (DummyPage, MI_PF_DUMMY_PAGE_PTE, 0);
//
// Give the page a containing frame so MiIdentifyPfn won't crash.
//
Pfn1->u4.PteFrame = PsInitialSystemProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT;
//
// Always bias the reference count by 1 and charge for this locked page
// up front so the myriad increments and decrements don't get slowed
// down with needless checking.
//
Pfn1->u3.e1.PrototypePte = 0;
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
Pfn1->u3.e1.ReadInProgress = 1;
MiReadInfo->DummyPagePfn = Pfn1;
DummyPfn1 = Pfn1;
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount + MdlPages);
//
// Properly initialize the inpage support block fields we overloaded.
//
InPageSupport->BasePte = *ProtoPteArray;
//
// Build the proper InPageSupport and MDL to describe this run.
//
for (; ProtoPteArray < EndProtoPteArray; ProtoPteArray += 1, IoPage += 1, ApiPage += 1) {
//
// Fill the MDL entry for this RLE.
//
PointerPte = *ProtoPteArray;
ASSERT (PointerPte != NULL);
//
// The PointerPte better be inside a prototype PTE allocation
// so that subsequent page trims update the correct PTEs.
//
ASSERT (((PointerPte >= (PMMPTE)MmPagedPoolStart) &&
(PointerPte <= (PMMPTE)MmPagedPoolEnd)) ||
((PointerPte >= (PMMPTE)MmSpecialPoolStart) && (PointerPte <= (PMMPTE)MmSpecialPoolEnd)));
//
// Check the state of this prototype PTE now that the PFN lock is held.
// If the page is not resident, the PTE must be put in transition with
// read in progress before the PFN lock is released.
//
//
// Lock page containing prototype PTEs in memory by
// incrementing the reference count for the page.
// Unlock any page locked earlier containing prototype PTEs if
// the containing page is not the same for both.
//
if (PfnProto != NULL) {
if (PointerPde != MiGetPteAddress (PointerPte)) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
PfnProto = NULL;
}
}
if (PfnProto == NULL) {
ASSERT (!MI_IS_PHYSICAL_ADDRESS (PointerPte));
PointerPde = MiGetPteAddress (PointerPte);
if (PointerPde->u.Hard.Valid == 0) {
MiMakeSystemAddressValidPfn (PointerPte, OldIrql);
}
PfnProto = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
MI_ADD_LOCKED_PAGE_CHARGE (PfnProto);
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
}
recheck:
PteContents = *PointerPte;
// LWFIX: are zero or dzero ptes possible here ?
ASSERT (PteContents.u.Long != 0);
if (PteContents.u.Hard.Valid == 1) {
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
*ApiPage = PageFrameIndex;
*IoPage = DummyPage;
continue;
}
if ((PteContents.u.Soft.Prototype == 0) &&
(PteContents.u.Soft.Transition == 1)) {
//
// The page is in transition. If there is an inpage still in
// progress, wait for it to complete. Reference the PFN and
// then march on.
//
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (&PteContents);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
if (Pfn1->u4.InPageError) {
//
// There was an in-page read error and there are other
// threads colliding for this page, delay to let the
// other threads complete and then retry.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
if (Pfn1->u3.e1.ReadInProgress) {
// LWFIX - start with temp\aw.c
}
//
// PTE refers to a normal transition PTE.
//
ASSERT ((SPFN_NUMBER)MmAvailablePages >= 0);
if (MmAvailablePages == 0) {
//
// This can only happen if the system is utilizing a hardware
// compression cache. This ensures that only a safe amount
// of the compressed virtual cache is directly mapped so that
// if the hardware gets into trouble, we can bail it out.
//
UNLOCK_PFN (OldIrql);
KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
LOCK_PFN (OldIrql);
goto recheck;
}
//
// The PFN reference count will be 1 already here if the
// modified writer has begun a write of this page. Otherwise
// it's ordinarily 0.
//
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
*IoPage = DummyPage;
*ApiPage = PageFrameIndex;
continue;
}
// LWFIX: need to handle protos that are now pagefile (or dzero)
// backed - prefetching it from the file here would cause us to lose
// the contents. Note this can happen for session-space images
// as we back modified (ie: for relocation fixups or IAT
// updated) portions from the pagefile. remove the assert below too.
ASSERT (PteContents.u.Soft.Prototype == 1);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (NULL, OldIrql))) {
//
// Had to wait so recheck all state.
//
goto recheck;
}
NumberOfPagesNeedingIo += 1;
//
// Allocate a physical page.
//
PageColor = MI_PAGE_COLOR_VA_PROCESS (
MiGetVirtualAddressMappedByPte (PointerPte),
&CurrentProcess->NextPageColor);
PageFrameIndex = MiRemoveAnyPage (PageColor);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (PointerPte->u.Hard.Valid == 0);
//
// Initialize read-in-progress PFN.
//
MiInitializePfn (PageFrameIndex, PointerPte, 0);
//
// These pieces of MiInitializePfn initialization are overridden
// here as these pages are only going into prototype
// transition and not into any page tables.
//
Pfn1->u3.e1.PrototypePte = 1;
Pfn1->u2.ShareCount -= 1;
ASSERT (Pfn1->u2.ShareCount == 0);
Pfn1->u3.e1.PageLocation = ZeroedPageList;
Pfn1->u3.e2.ReferenceCount -= 1;
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);
//
// Initialize the I/O specific fields.
//
Pfn1->u1.Event = &InPageSupport->Event;
Pfn1->u3.e1.ReadInProgress = 1;
ASSERT (Pfn1->u4.InPageError == 0);
//
// Increment the PFN reference count in the control area for
// the subsection.
//
MiReadInfo->ControlArea->NumberOfPfnReferences += 1;
//
// Put the prototype PTE into the transition state.
//
MI_MAKE_TRANSITION_PTE (TempPte,
PageFrameIndex,
PointerPte->u.Soft.Protection,
PointerPte);
MI_WRITE_INVALID_PTE (PointerPte, TempPte);
*IoPage = PageFrameIndex;
*ApiPage = PageFrameIndex;
}
//
// If all the pages were resident, dereference the dummy page references
// now and notify our caller that I/O is not necessary.
//
if (NumberOfPagesNeedingIo == 0) {
ASSERT (DummyPfn1->u3.e2.ReferenceCount > MdlPages);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - MdlPages);
//
// Unlock page containing prototype PTEs.
//
if (PfnProto != NULL) {
ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
}
UNLOCK_PFN (OldIrql);
//
// Return the upfront resident available charge as the
// individual charges have all been made at this point.
//
MI_INCREMENT_RESIDENT_AVAILABLE (ResidentAvailableCharge,
MM_RESAVAIL_FREE_BUILDMDL_EXCESS);
return STATUS_SUCCESS;
}
//
// Carefully trim leading dummy pages.
//
Page = (PPFN_NUMBER)(Mdl + 1);
DummyTrim = 0;
for (i = 0; i < MdlPages - 1; i += 1) {
if (*Page == DummyPage) {
DummyTrim += 1;
Page += 1;
}
else {
break;
}
}
if (DummyTrim != 0) {
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
ASSERT (Mdl->ByteCount != 0);
InPageSupport->ReadOffset.QuadPart += (DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
//
// Shuffle down the PFNs in the MDL.
// Recalculate BasePte to adjust for the shuffle.
//
Pfn1 = MI_PFN_ELEMENT (*Page);
ASSERT (Pfn1->PteAddress->u.Hard.Valid == 0);
ASSERT ((Pfn1->PteAddress->u.Soft.Prototype == 0) &&
(Pfn1->PteAddress->u.Soft.Transition == 1));
InPageSupport->BasePte = Pfn1->PteAddress;
DestinationPage = (PPFN_NUMBER)(Mdl + 1);
do {
*DestinationPage = *Page;
DestinationPage += 1;
Page += 1;
i += 1;
} while (i < MdlPages);
MdlPages -= DummyTrim;
}
//
// Carefully trim trailing dummy pages.
//
ASSERT (MdlPages != 0);
Page = (PPFN_NUMBER)(Mdl + 1) + MdlPages - 1;
if (*Page == DummyPage) {
ASSERT (MdlPages >= 2);
//
// Trim the last page specially as it may be a partial page.
//
Mdl->Size -= sizeof(PFN_NUMBER);
if (BYTE_OFFSET(Mdl->ByteCount) != 0) {
Mdl->ByteCount &= ~(PAGE_SIZE - 1);
}
else {
Mdl->ByteCount -= PAGE_SIZE;
}
ASSERT (Mdl->ByteCount != 0);
DummyPfn1->u3.e2.ReferenceCount -= 1;
//
// Now trim any other trailing pages.
//
Page -= 1;
DummyTrim = 0;
while (Page != ((PPFN_NUMBER)(Mdl + 1))) {
if (*Page != DummyPage) {
break;
}
DummyTrim += 1;
Page -= 1;
}
if (DummyTrim != 0) {
ASSERT (Mdl->Size > (USHORT)(DummyTrim * sizeof(PFN_NUMBER)));
Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
DummyPfn1->u3.e2.ReferenceCount =
(USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
}
ASSERT (MdlPages > DummyTrim + 1);
MdlPages -= (DummyTrim + 1);
#if DBG
StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
ASSERT (MdlPages == ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa,
Mdl->ByteCount));
#endif
}
//
// If the MDL is not already embedded in the inpage block, see if its
// final size qualifies it - if so, embed it now.
//
if ((Mdl != &InPageSupport->Mdl) &&
(Mdl->ByteCount <= (MM_MAXIMUM_READ_CLUSTER_SIZE + 1) * PAGE_SIZE)){
#if DBG
RtlFillMemoryUlong (&InPageSupport->Page[0],
(MM_MAXIMUM_READ_CLUSTER_SIZE+1) * sizeof (PFN_NUMBER),
0xf1f1f1f1);
#endif
RtlCopyMemory (&InPageSupport->Mdl, Mdl, Mdl->Size);
FreeMdl = Mdl;
Mdl = &InPageSupport->Mdl;
ASSERT (((ULONG_PTR)Mdl & (sizeof(QUAD) - 1)) == 0);
InPageSupport->u1.e1.PrefetchMdlHighBits = ((ULONG_PTR)Mdl >> 3);
}
VOID
MiCheckPfn (
)
/*++
Routine Description:
This routine checks each physical page in the PFN database to ensure
it is in the proper state.
Arguments:
None.
Return Value:
None.
Environment:
Kernel mode, APCs disabled.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER Link, Previous;
ULONG i;
PMMPTE PointerPte;
KIRQL PreviousIrql;
KIRQL OldIrql;
USHORT ValidCheck[4];
USHORT ValidPage[4];
PMMPFN PfnX;
ValidCheck[0] = ValidCheck[1] = ValidCheck[2] = ValidCheck[3] = 0;
ValidPage[0] = ValidPage[1] = ValidPage[2] = ValidPage[3] = 0;
if (CheckPfnBitMap == NULL) {
MiCreateBitMap ( &CheckPfnBitMap, MmNumberOfPhysicalPages, NonPagedPool);
}
RtlClearAllBits (CheckPfnBitMap);
//
// Walk free list.
//
KeRaiseIrql (APC_LEVEL, &PreviousIrql);
LOCK_PFN (OldIrql);
Previous = MM_EMPTY_LIST;
Link = MmFreePageListHead.Flink;
for (i=0; i < MmFreePageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on free list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != FreePageList) {
DbgPrint("page location not freelist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on free list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk zeroed list.
//
Previous = MM_EMPTY_LIST;
Link = MmZeroedPageListHead.Flink;
for (i=0; i < MmZeroedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on zero list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ZeroedPageList) {
DbgPrint("page location not zerolist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on zero list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Bad list.
//
Previous = MM_EMPTY_LIST;
Link = MmBadPageListHead.Flink;
for (i=0; i < MmBadPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Bad list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != BadPageList) {
DbgPrint("page location not Badlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Bad list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Standby list.
//
Previous = MM_EMPTY_LIST;
Link = MmStandbyPageListHead.Flink;
for (i=0; i < MmStandbyPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Standby list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != StandbyPageList) {
DbgPrint("page location not Standbylist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Standby list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in standby list\n");
MiFormatPfn(Pfn1);
} else {
OldIrql = 99;
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame,
&OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Modified list.
//
Previous = MM_EMPTY_LIST;
Link = MmModifiedPageListHead.Flink;
for (i=0; i < MmModifiedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Modified list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ModifiedPageList) {
DbgPrint("page location not Modifiedlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Modified list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in modified list\n");
MiFormatPfn(Pfn1);
} else {
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// All non active pages have been scanned. Locate the
// active pages and make sure they are consistent.
//
//
// set bit zero as page zero is reserved for now
//
RtlSetBits (CheckPfnBitMap, 0L, 1L);
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
while (Link != 0xFFFFFFFF) {
Pfn1 = MI_PFN_ELEMENT (Link);
//
// Make sure the PTE address is okay
//
if ((Pfn1->PteAddress >= (PMMPTE)HYPER_SPACE)
&& (Pfn1->u3.e1.PrototypePte == 0)) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
if (Pfn1->PteAddress < (PMMPTE)PTE_BASE) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
#if defined(_IA64_)
//
// ignore PTEs mapped to IA64 kernel BAT.
//
if (MI_IS_PHYSICAL_ADDRESS(MiGetVirtualAddressMappedByPte(Pfn1->PteAddress))) {
goto NoCheck;
}
#endif // _IA64_
#ifdef _ALPHA_
//
// ignore ptes mapped to ALPHA's 32-bit superpage.
//
if ((Pfn1->PteAddress > (PMMPTE)(ULONG_PTR)0xc0100000) &&
(Pfn1->PteAddress < (PMMPTE)(ULONG_PTR)0xc0180000)) {
goto NoCheck;
}
#endif //ALPHA
//
// Check to make sure the referenced PTE is for this page.
//
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Hard.Valid == 0) {
//
// if the page is a page table page it could be out of
// the working set yet a transition page is keeping it
// around in memory (ups the share count).
//
if ((Pfn1->PteAddress < (PMMPTE)PDE_BASE) ||
(Pfn1->PteAddress > (PMMPTE)PDE_TOP)) {
DbgPrint("Pte not valid for page on active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
}
if (Pfn1->u3.e2.ReferenceCount != 1) {
DbgPrint("refcount not 1\n");
MiFormatPfn(Pfn1);
}
//
// Check to make sure the PTE count for the frame is okay.
//
if (Pfn1->u3.e1.PrototypePte == 1) {
PfnX = MI_PFN_ELEMENT(Pfn1->PteFrame);
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
ValidPage[i] = (USHORT)Pfn1->PteFrame;
}
if (ValidPage[i] == (USHORT)Pfn1->PteFrame) {
ValidCheck[i] += 1;
break;
}
}
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
#if defined(_ALPHA_) || defined(_IA64_)
NoCheck:
#endif
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
}
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
break;
}
PfnX = MI_PFN_ELEMENT(ValidPage[i]);
}
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
ULONG
MiDecommitPages (
IN PVOID StartingAddress,
IN PMMPTE EndingPte,
IN PEPROCESS Process,
IN PMMVAD_SHORT Vad
)
/*++
Routine Description:
This routine decommits the specified range of pages.
Arguments:
StartingAddress - Supplies the starting address of the range.
EndingPte - Supplies the ending PTE of the range.
Process - Supplies the current process.
Vad - Supplies the virtual address descriptor which describes the range.
Return Value:
Value to reduce commitment by for the VAD.
Environment:
Kernel mode, APCs disabled, AddressCreation mutex held.
--*/
{
PMMPTE PointerPde;
PMMPTE PointerPte;
PVOID Va;
ULONG CommitReduction;
PMMPTE CommitLimitPte;
KIRQL OldIrql;
PMMPTE ValidPteList[MM_VALID_PTE_SIZE];
ULONG count;
WSLE_NUMBER WorkingSetIndex;
PMMPFN Pfn1;
PMMPFN Pfn2;
WSLE_NUMBER Entry;
MMWSLENTRY Locked;
MMPTE PteContents;
PFN_NUMBER PageTableFrameIndex;
PVOID UsedPageTableHandle;
PETHREAD CurrentThread;
count = 0;
CommitReduction = 0;
if (Vad->u.VadFlags.MemCommit) {
CommitLimitPte = MiGetPteAddress (MI_VPN_TO_VA (Vad->EndingVpn));
}
else {
CommitLimitPte = NULL;
}
//
// Decommit each page by setting the PTE to be explicitly
// decommitted. The PTEs cannot be deleted all at once as
// this would set the PTEs to zero which would auto-evaluate
// as committed if referenced by another thread when a page
// table page is being in-paged.
//
PointerPde = MiGetPdeAddress (StartingAddress);
PointerPte = MiGetPteAddress (StartingAddress);
Va = StartingAddress;
//
// Loop through all the PDEs which map this region and ensure that
// they exist. If they don't exist create them by touching a
// PTE mapped by the PDE.
//
CurrentThread = PsGetCurrentThread ();
LOCK_WS_UNSAFE (CurrentThread, Process);
MiMakePdeExistAndMakeValid (PointerPde, Process, MM_NOIRQL);
while (PointerPte <= EndingPte) {
if (MiIsPteOnPdeBoundary (PointerPte)) {
PointerPde = MiGetPdeAddress (Va);
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
MiMakePdeExistAndMakeValid (PointerPde, Process, MM_NOIRQL);
}
//
// The working set lock is held. No PTEs can go from
// invalid to valid or valid to invalid. Transition
// PTEs can go from transition to pagefile.
//
PteContents = *PointerPte;
if (PteContents.u.Long != 0) {
if (PointerPte->u.Long == MmDecommittedPte.u.Long) {
//
// This PTE is already decommitted.
//
CommitReduction += 1;
}
else {
Process->NumberOfPrivatePages -= 1;
if (PteContents.u.Hard.Valid == 1) {
//
// Make sure this is not a forked PTE.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);
if (Pfn1->u3.e1.PrototypePte) {
//
// MiDeletePte may release both the working set pushlock
// and the PFN lock so the valid PTE list must be
// processed now.
//
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL,
OldIrql);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
else {
//
// PTE is valid, process later when PFN lock is held.
//
if (count == MM_VALID_PTE_SIZE) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
ValidPteList[count] = PointerPte;
count += 1;
//
// Remove address from working set list.
//
WorkingSetIndex = Pfn1->u1.WsIndex;
ASSERT (PAGE_ALIGN(MmWsle[WorkingSetIndex].u1.Long) ==
Va);
//
// Check to see if this entry is locked in the
// working set or locked in memory.
//
Locked = MmWsle[WorkingSetIndex].u1.e1;
MiRemoveWsle (WorkingSetIndex, MmWorkingSetList);
//
// Add this entry to the list of free working set
// entries and adjust the working set count.
//
MiReleaseWsle (WorkingSetIndex, &Process->Vm);
if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {
//
// This entry is locked.
//
MmWorkingSetList->FirstDynamic -= 1;
if (WorkingSetIndex != MmWorkingSetList->FirstDynamic) {
Entry = MmWorkingSetList->FirstDynamic;
ASSERT (MmWsle[Entry].u1.e1.Valid);
MiSwapWslEntries (Entry,
WorkingSetIndex,
&Process->Vm,
FALSE);
}
}
MI_SET_PTE_IN_WORKING_SET (PointerPte, 0);
}
}
else if (PteContents.u.Soft.Prototype) {
//
// This is a forked PTE, just delete it.
//
// MiDeletePte may release both the working set pushlock
// and the PFN lock so the valid PTE list must be
// processed now.
//
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
count = 0;
}
LOCK_PFN (OldIrql);
MiDeletePte (PointerPte,
Va,
FALSE,
Process,
NULL,
NULL,
OldIrql);
UNLOCK_PFN (OldIrql);
Process->NumberOfPrivatePages += 1;
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
else if (PteContents.u.Soft.Transition == 1) {
//
// Transition PTE, get the PFN database lock
// and reprocess this one.
//
LOCK_PFN (OldIrql);
PteContents = *PointerPte;
if (PteContents.u.Soft.Transition == 1) {
//
// PTE is still in transition, delete it.
//
Pfn1 = MI_PFN_ELEMENT (PteContents.u.Trans.PageFrameNumber);
MI_SET_PFN_DELETED (Pfn1);
PageTableFrameIndex = Pfn1->u4.PteFrame;
Pfn2 = MI_PFN_ELEMENT (PageTableFrameIndex);
MiDecrementShareCountInline (Pfn2, PageTableFrameIndex);
//
// Check the reference count for the page, if the
// reference count is zero, move the page to the
// free list, if the reference count is not zero,
// ignore this page. When the reference count
// goes to zero, it will be placed on the free list.
//
if (Pfn1->u3.e2.ReferenceCount == 0) {
MiUnlinkPageFromList (Pfn1);
MiReleasePageFileSpace (Pfn1->OriginalPte);
MiInsertPageInFreeList (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE(&PteContents));
}
}
else {
//
// Page MUST be in page file format!
//
ASSERT (PteContents.u.Soft.Valid == 0);
ASSERT (PteContents.u.Soft.Prototype == 0);
ASSERT (PteContents.u.Soft.PageFileHigh != 0);
MiReleasePageFileSpace (PteContents);
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
UNLOCK_PFN (OldIrql);
}
else {
//
// Must be demand zero or paging file format.
//
if (PteContents.u.Soft.PageFileHigh != 0) {
LOCK_PFN (OldIrql);
MiReleasePageFileSpace (PteContents);
UNLOCK_PFN (OldIrql);
}
else {
//
// Don't subtract out the private page count for
// a demand zero page.
//
Process->NumberOfPrivatePages += 1;
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
}
}
else {
//
// The PTE is already zero.
//
//
// Increment the count of non-zero page table entries for this
// page table and the number of private pages for the process.
//
UsedPageTableHandle = MI_GET_USED_PTES_HANDLE (Va);
MI_INCREMENT_USED_PTES_BY_HANDLE (UsedPageTableHandle);
if (PointerPte > CommitLimitPte) {
//
// PTE is not committed.
//
CommitReduction += 1;
}
MI_WRITE_INVALID_PTE (PointerPte, MmDecommittedPte);
}
PointerPte += 1;
Va = (PVOID)((PCHAR)Va + PAGE_SIZE);
}
if (count != 0) {
MiProcessValidPteList (&ValidPteList[0], count);
}
UNLOCK_WS_UNSAFE (CurrentThread, Process);
return CommitReduction;
}
