NTSTATUS
NTAPI
MiCopyPageToPage(PFN_NUMBER DestPage, PFN_NUMBER SrcPage)
{
PEPROCESS Process;
KIRQL Irql, Irql2;
PVOID TempAddress, TempSource;
Process = PsGetCurrentProcess();
TempAddress = MiMapPageInHyperSpace(Process, DestPage, &Irql);
if (TempAddress == NULL)
{
return STATUS_NO_MEMORY;
}
TempSource = MiMapPageInHyperSpace(Process, SrcPage, &Irql2);
if (!TempSource) {
MiUnmapPageInHyperSpace(Process, TempAddress, Irql);
return STATUS_NO_MEMORY;
}
memcpy(TempAddress, TempSource, PAGE_SIZE);
MiUnmapPageInHyperSpace(Process, TempSource, Irql2);
MiUnmapPageInHyperSpace(Process, TempAddress, Irql);
return STATUS_SUCCESS;
}
NTSTATUS
NTAPI
_MiWriteBackPage(PFILE_OBJECT FileObject,
PLARGE_INTEGER FileOffset,
ULONG Length,
PFN_NUMBER Page,
const char *File,
int Line)
{
NTSTATUS Status;
PVOID Hyperspace;
IO_STATUS_BLOCK Iosb;
KIRQL OldIrql;
PVOID PageBuffer = ExAllocatePool(NonPagedPool, PAGE_SIZE);
if (!PageBuffer) return STATUS_NO_MEMORY;
Hyperspace = MiMapPageInHyperSpace(PsGetCurrentProcess(), Page, &OldIrql);
if (!Hyperspace)
{
ExFreePool(PageBuffer);
return STATUS_NO_MEMORY;
}
RtlCopyMemory(PageBuffer, Hyperspace, PAGE_SIZE);
MiUnmapPageInHyperSpace(PsGetCurrentProcess(), Hyperspace, OldIrql);
DPRINT("MiWriteBackPage(%wZ,%08x%08x,%s:%d)\n",
&FileObject->FileName,
FileOffset->u.HighPart,
FileOffset->u.LowPart,
File,
Line);
Status = MiSimpleWrite(FileObject,
FileOffset,
PageBuffer,
Length,
&Iosb);
ExFreePool(PageBuffer);
if (!NT_SUCCESS(Status))
{
DPRINT1("MiSimpleWrite failed (%x)\n", Status);
}
return Status;
}
VOID
MiCheckPfn (
)
/*++
Routine Description:
This routine checks each physical page in the PFN database to ensure
it is in the proper state.
Arguments:
None.
Return Value:
None.
Environment:
Kernel mode, APCs disabled.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER Link, Previous;
ULONG i;
PMMPTE PointerPte;
KIRQL PreviousIrql;
KIRQL OldIrql;
USHORT ValidCheck[4];
USHORT ValidPage[4];
PMMPFN PfnX;
ValidCheck[0] = ValidCheck[1] = ValidCheck[2] = ValidCheck[3] = 0;
ValidPage[0] = ValidPage[1] = ValidPage[2] = ValidPage[3] = 0;
if (CheckPfnBitMap == NULL) {
MiCreateBitMap ( &CheckPfnBitMap, MmNumberOfPhysicalPages, NonPagedPool);
}
RtlClearAllBits (CheckPfnBitMap);
//
// Walk free list.
//
KeRaiseIrql (APC_LEVEL, &PreviousIrql);
LOCK_PFN (OldIrql);
Previous = MM_EMPTY_LIST;
Link = MmFreePageListHead.Flink;
for (i=0; i < MmFreePageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on free list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != FreePageList) {
DbgPrint("page location not freelist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on free list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("free list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk zeroed list.
//
Previous = MM_EMPTY_LIST;
Link = MmZeroedPageListHead.Flink;
for (i=0; i < MmZeroedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on zero list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ZeroedPageList) {
DbgPrint("page location not zerolist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on zero list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("zero list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Bad list.
//
Previous = MM_EMPTY_LIST;
Link = MmBadPageListHead.Flink;
for (i=0; i < MmBadPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Bad list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != BadPageList) {
DbgPrint("page location not Badlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Bad list\n");
MiFormatPfn(Pfn1);
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Bad list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Standby list.
//
Previous = MM_EMPTY_LIST;
Link = MmStandbyPageListHead.Flink;
for (i=0; i < MmStandbyPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Standby list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != StandbyPageList) {
DbgPrint("page location not Standbylist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Standby list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in standby list\n");
MiFormatPfn(Pfn1);
} else {
OldIrql = 99;
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame,
&OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on standby list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Standby list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// Walk Modified list.
//
Previous = MM_EMPTY_LIST;
Link = MmModifiedPageListHead.Flink;
for (i=0; i < MmModifiedPageListHead.Total; i++) {
if (Link == MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
Pfn1 = MI_PFN_ELEMENT(Link);
if (Pfn1->u3.e2.ReferenceCount != 0) {
DbgPrint("non zero reference count on Modified list\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u3.e1.PageLocation != ModifiedPageList) {
DbgPrint("page location not Modifiedlist\n");
MiFormatPfn(Pfn1);
}
if (Pfn1->u2.Blink != Previous) {
DbgPrint("bad blink on Modified list\n");
MiFormatPfn(Pfn1);
}
//
// Check to see if referenced PTE is okay.
//
if (MI_IS_PFN_DELETED (Pfn1)) {
DbgPrint("Invalid pteaddress in modified list\n");
MiFormatPfn(Pfn1);
} else {
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Soft.Transition == 0) {
DbgPrint("Pte not in transition for page on modified list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
}
Previous = Link;
Link = Pfn1->u1.Flink;
}
if (Link != MM_EMPTY_LIST) {
DbgPrint("Modified list total count wrong\n");
Pfn1 = MI_PFN_ELEMENT(Link);
MiFormatPfn(Pfn1);
}
//
// All non active pages have been scanned. Locate the
// active pages and make sure they are consistent.
//
//
// set bit zero as page zero is reserved for now
//
RtlSetBits (CheckPfnBitMap, 0L, 1L);
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
while (Link != 0xFFFFFFFF) {
Pfn1 = MI_PFN_ELEMENT (Link);
//
// Make sure the PTE address is okay
//
if ((Pfn1->PteAddress >= (PMMPTE)HYPER_SPACE)
&& (Pfn1->u3.e1.PrototypePte == 0)) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
if (Pfn1->PteAddress < (PMMPTE)PTE_BASE) {
DbgPrint("pfn with illegal pte address\n");
MiFormatPfn(Pfn1);
break;
}
#if defined(_IA64_)
//
// ignore PTEs mapped to IA64 kernel BAT.
//
if (MI_IS_PHYSICAL_ADDRESS(MiGetVirtualAddressMappedByPte(Pfn1->PteAddress))) {
goto NoCheck;
}
#endif // _IA64_
#ifdef _ALPHA_
//
// ignore ptes mapped to ALPHA's 32-bit superpage.
//
if ((Pfn1->PteAddress > (PMMPTE)(ULONG_PTR)0xc0100000) &&
(Pfn1->PteAddress < (PMMPTE)(ULONG_PTR)0xc0180000)) {
goto NoCheck;
}
#endif //ALPHA
//
// Check to make sure the referenced PTE is for this page.
//
if ((Pfn1->u3.e1.PrototypePte == 1) &&
(MmIsAddressValid (Pfn1->PteAddress))) {
PointerPte = Pfn1->PteAddress;
} else {
PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
MiGetByteOffset(Pfn1->PteAddress));
}
if (MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) != Link) {
DbgPrint("Invalid PFN - PTE address is wrong in active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
if (PointerPte->u.Hard.Valid == 0) {
//
// if the page is a page table page it could be out of
// the working set yet a transition page is keeping it
// around in memory (ups the share count).
//
if ((Pfn1->PteAddress < (PMMPTE)PDE_BASE) ||
(Pfn1->PteAddress > (PMMPTE)PDE_TOP)) {
DbgPrint("Pte not valid for page on active list\n");
MiFormatPfn(Pfn1);
MiFormatPte(PointerPte);
}
}
if (Pfn1->u3.e2.ReferenceCount != 1) {
DbgPrint("refcount not 1\n");
MiFormatPfn(Pfn1);
}
//
// Check to make sure the PTE count for the frame is okay.
//
if (Pfn1->u3.e1.PrototypePte == 1) {
PfnX = MI_PFN_ELEMENT(Pfn1->PteFrame);
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
ValidPage[i] = (USHORT)Pfn1->PteFrame;
}
if (ValidPage[i] == (USHORT)Pfn1->PteFrame) {
ValidCheck[i] += 1;
break;
}
}
}
if (OldIrql != 99) {
MiUnmapPageInHyperSpace (OldIrql);
OldIrql = 99;
}
#if defined(_ALPHA_) || defined(_IA64_)
NoCheck:
#endif
Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
}
for (i = 0; i < 4; i++) {
if (ValidPage[i] == 0) {
break;
}
PfnX = MI_PFN_ELEMENT(ValidPage[i]);
}
UNLOCK_PFN (OldIrql);
KeLowerIrql (PreviousIrql);
return;
}
BOOLEAN
MmCreateProcessAddressSpace (
IN ULONG MinimumWorkingSetSize,
IN PEPROCESS NewProcess,
OUT PULONG_PTR DirectoryTableBase
)
/*++
Routine Description:
This routine creates an address space which maps the system
portion and contains a hyper space entry.
Arguments:
MinimumWorkingSetSize - Supplies the minimum working set size for
this address space. This value is only used
to ensure that ample physical pages exist
to create this process.
NewProcess - Supplies a pointer to the process object being created.
DirectoryTableBase - Returns the value of the newly created
address space's Page Directory (PD) page and
hyper space page.
Return Value:
Returns TRUE if an address space was successfully created, FALSE
if ample physical pages do not exist.
Environment:
Kernel mode. APCs Disabled.
--*/
{
LOGICAL FlushTbNeeded;
PFN_NUMBER PageDirectoryIndex;
PFN_NUMBER HyperSpaceIndex;
PFN_NUMBER PageContainingWorkingSet;
PFN_NUMBER VadBitMapPage;
MMPTE TempPte;
MMPTE TempPte2;
PEPROCESS CurrentProcess;
KIRQL OldIrql;
PMMPFN Pfn1;
ULONG Color;
PMMPTE PointerPte;
ULONG PdeOffset;
PMMPTE MappingPte;
PMMPTE PointerFillPte;
PMMPTE CurrentAddressSpacePde;
//
// Charge commitment for the page directory pages, working set page table
// page, and working set list. If Vad bitmap lookups are enabled, then
// charge for a page or two for that as well.
//
if (MiChargeCommitment (MM_PROCESS_COMMIT_CHARGE, NULL) == FALSE) {
return FALSE;
}
FlushTbNeeded = FALSE;
CurrentProcess = PsGetCurrentProcess ();
NewProcess->NextPageColor = (USHORT) (RtlRandom (&MmProcessColorSeed));
KeInitializeSpinLock (&NewProcess->HyperSpaceLock);
//
// Get the PFN lock to get physical pages.
//
LOCK_PFN (OldIrql);
//
// Check to make sure the physical pages are available.
//
if (MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MinimumWorkingSetSize){
UNLOCK_PFN (OldIrql);
MiReturnCommitment (MM_PROCESS_COMMIT_CHARGE);
//
// Indicate no directory base was allocated.
//
return FALSE;
}
MM_TRACK_COMMIT (MM_DBG_COMMIT_PROCESS_CREATE, MM_PROCESS_COMMIT_CHARGE);
MI_DECREMENT_RESIDENT_AVAILABLE (MinimumWorkingSetSize,
MM_RESAVAIL_ALLOCATE_CREATE_PROCESS);
//
// Allocate a page directory page.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_PTE_PROCESS (PDE_BASE,
&CurrentProcess->NextPageColor);
PageDirectoryIndex = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (PageDirectoryIndex);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Allocate the hyper space page table page.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_PTE_PROCESS (MiGetPdeAddress(HYPER_SPACE),
&CurrentProcess->NextPageColor);
HyperSpaceIndex = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (HyperSpaceIndex);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Remove page(s) for the VAD bitmap.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_VA_PROCESS (MmWorkingSetList,
&CurrentProcess->NextPageColor);
VadBitMapPage = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (VadBitMapPage);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
//
// Remove a page for the working set list.
//
if (MmAvailablePages < MM_HIGH_LIMIT) {
MiEnsureAvailablePageOrWait (NULL, OldIrql);
}
Color = MI_PAGE_COLOR_VA_PROCESS (MmWorkingSetList,
&CurrentProcess->NextPageColor);
PageContainingWorkingSet = MiRemoveZeroPageMayReleaseLocks (Color, OldIrql);
Pfn1 = MI_PFN_ELEMENT (PageContainingWorkingSet);
if (Pfn1->u3.e1.CacheAttribute != MiCached) {
Pfn1->u3.e1.CacheAttribute = MiCached;
FlushTbNeeded = TRUE;
}
UNLOCK_PFN (OldIrql);
if (FlushTbNeeded == TRUE) {
MI_FLUSH_TB_FOR_CACHED_ATTRIBUTE ();
}
ASSERT (NewProcess->AddressSpaceInitialized == 0);
PS_SET_BITS (&NewProcess->Flags, PS_PROCESS_FLAGS_ADDRESS_SPACE1);
ASSERT (NewProcess->AddressSpaceInitialized == 1);
NewProcess->Vm.MinimumWorkingSetSize = MinimumWorkingSetSize;
NewProcess->WorkingSetPage = PageContainingWorkingSet;
INITIALIZE_DIRECTORY_TABLE_BASE (&DirectoryTableBase[0], PageDirectoryIndex);
INITIALIZE_DIRECTORY_TABLE_BASE (&DirectoryTableBase[1], HyperSpaceIndex);
//
// Initialize the page reserved for hyper space.
//
TempPte = ValidPdePde;
MI_SET_GLOBAL_STATE (TempPte, 0);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
HyperSpaceIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
PointerPte = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
PointerPte = MiMapPageInHyperSpace (CurrentProcess, HyperSpaceIndex, &OldIrql);
}
TempPte.u.Hard.PageFrameNumber = VadBitMapPage;
PointerPte[MiGetPteOffset(VAD_BITMAP_SPACE)] = TempPte;
TempPte.u.Hard.PageFrameNumber = PageContainingWorkingSet;
PointerPte[MiGetPteOffset(MmWorkingSetList)] = TempPte;
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, PointerPte, OldIrql);
}
//
// Set the PTE address in the PFN for the page directory page.
//
Pfn1 = MI_PFN_ELEMENT (PageDirectoryIndex);
Pfn1->PteAddress = (PMMPTE)PDE_BASE;
TempPte = ValidPdePde;
TempPte.u.Hard.PageFrameNumber = HyperSpaceIndex;
MI_SET_GLOBAL_STATE (TempPte, 0);
//
// Add the new process to our internal list prior to filling any
// system PDEs so if a system PDE changes (large page map or unmap)
// it can mark this process for a subsequent update.
//
ASSERT (NewProcess->Pcb.DirectoryTableBase[0] == 0);
LOCK_EXPANSION (OldIrql);
InsertTailList (&MmProcessList, &NewProcess->MmProcessLinks);
UNLOCK_EXPANSION (OldIrql);
//
// Map the page directory page in hyperspace.
//
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
PageDirectoryIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
PointerPte = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
PointerPte = MiMapPageInHyperSpace (CurrentProcess, PageDirectoryIndex, &OldIrql);
}
PdeOffset = MiGetPdeOffset (MmSystemRangeStart);
PointerFillPte = &PointerPte[PdeOffset];
CurrentAddressSpacePde = MiGetPdeAddress (MmSystemRangeStart);
RtlCopyMemory (PointerFillPte,
CurrentAddressSpacePde,
PAGE_SIZE - PdeOffset * sizeof (MMPTE));
//
// Map the working set page table page.
//
PdeOffset = MiGetPdeOffset (HYPER_SPACE);
PointerPte[PdeOffset] = TempPte;
//
// Zero the remaining page directory range used to map the working
// set list and its hash.
//
PdeOffset += 1;
ASSERT (MiGetPdeOffset (MmHyperSpaceEnd) >= PdeOffset);
MiZeroMemoryPte (&PointerPte[PdeOffset],
(MiGetPdeOffset (MmHyperSpaceEnd) - PdeOffset + 1));
//
// Recursively map the page directory page so it points to itself.
//
TempPte.u.Hard.PageFrameNumber = PageDirectoryIndex;
PointerPte[MiGetPdeOffset(PTE_BASE)] = TempPte;
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, PointerPte, OldIrql);
}
InterlockedExchangeAddSizeT (&MmProcessCommit, MM_PROCESS_COMMIT_CHARGE);
//
// Up the session space reference count.
//
MiSessionAddProcess (NewProcess);
return TRUE;
}
NTSTATUS
NTAPI
MiReadFilePage(PMMSUPPORT AddressSpace,
PMEMORY_AREA MemoryArea,
PMM_REQUIRED_RESOURCES RequiredResources)
{
PFILE_OBJECT FileObject = RequiredResources->Context;
PPFN_NUMBER Page = &RequiredResources->Page[RequiredResources->Offset];
PLARGE_INTEGER FileOffset = &RequiredResources->FileOffset;
NTSTATUS Status;
PVOID PageBuf = NULL;
KEVENT Event;
IO_STATUS_BLOCK IOSB;
UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)];
PMDL Mdl = (PMDL)MdlBase;
KIRQL OldIrql;
DPRINTC("Pulling page %I64x from %wZ to %Ix\n",
FileOffset->QuadPart,
&FileObject->FileName,
*Page);
Status = MmRequestPageMemoryConsumer(RequiredResources->Consumer,
TRUE,
Page);
if (!NT_SUCCESS(Status))
{
DPRINT1("Status: %x\n", Status);
return Status;
}
MmInitializeMdl(Mdl, NULL, PAGE_SIZE);
MmBuildMdlFromPages(Mdl, Page);
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
KeInitializeEvent(&Event, NotificationEvent, FALSE);
Status = IoPageRead(FileObject, Mdl, FileOffset, &Event, &IOSB);
if (Status == STATUS_PENDING)
{
KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
Status = IOSB.Status;
}
if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
{
MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
}
PageBuf = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql);
if (!PageBuf)
{
MmReleasePageMemoryConsumer(RequiredResources->Consumer, *Page);
return STATUS_NO_MEMORY;
}
RtlZeroMemory((PCHAR)PageBuf+RequiredResources->Amount,
PAGE_SIZE-RequiredResources->Amount);
MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageBuf, OldIrql);
DPRINT("Read Status %x (Page %x)\n", Status, *Page);
if (!NT_SUCCESS(Status))
{
MmReleasePageMemoryConsumer(RequiredResources->Consumer, *Page);
DPRINT("Status: %x\n", Status);
return Status;
}
return STATUS_SUCCESS;
}
LOGICAL
FASTCALL
MiCopyOnWrite (
IN PVOID FaultingAddress,
IN PMMPTE PointerPte
)
/*++
Routine Description:
This routine performs a copy on write operation for the specified
virtual address.
Arguments:
FaultingAddress - Supplies the virtual address which caused the fault.
PointerPte - Supplies the pointer to the PTE which caused the page fault.
Return Value:
Returns TRUE if the page was actually split, FALSE if not.
Environment:
Kernel mode, APCs disabled, working set mutex held.
--*/
{
MMPTE TempPte;
MMPTE TempPte2;
PMMPTE MappingPte;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER NewPageIndex;
PVOID CopyTo;
PVOID CopyFrom;
KIRQL OldIrql;
PMMPFN Pfn1;
PEPROCESS CurrentProcess;
PMMCLONE_BLOCK CloneBlock;
PMMCLONE_DESCRIPTOR CloneDescriptor;
WSLE_NUMBER WorkingSetIndex;
LOGICAL FakeCopyOnWrite;
PMMWSL WorkingSetList;
PVOID SessionSpace;
PLIST_ENTRY NextEntry;
PIMAGE_ENTRY_IN_SESSION Image;
//
// This is called from MmAccessFault, the PointerPte is valid
// and the working set mutex ensures it cannot change state.
//
// Capture the PTE contents to TempPte.
//
TempPte = *PointerPte;
ASSERT (TempPte.u.Hard.Valid == 1);
PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
//
// Check to see if this is a prototype PTE with copy on write enabled.
//
FakeCopyOnWrite = FALSE;
CurrentProcess = PsGetCurrentProcess ();
CloneBlock = NULL;
if (FaultingAddress >= (PVOID) MmSessionBase) {
WorkingSetList = MmSessionSpace->Vm.VmWorkingSetList;
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
SessionSpace = (PVOID) MmSessionSpace;
MM_SESSION_SPACE_WS_LOCK_ASSERT ();
if (MmSessionSpace->ImageLoadingCount != 0) {
NextEntry = MmSessionSpace->ImageList.Flink;
while (NextEntry != &MmSessionSpace->ImageList) {
Image = CONTAINING_RECORD (NextEntry, IMAGE_ENTRY_IN_SESSION, Link);
if ((FaultingAddress >= Image->Address) &&
(FaultingAddress <= Image->LastAddress)) {
if (Image->ImageLoading) {
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
TempPte.u.Hard.CopyOnWrite = 0;
TempPte.u.Hard.Write = 1;
//
// The page is no longer copy on write, update the PTE
// setting both the dirty bit and the accessed bit.
//
// Even though the page's current backing is the image
// file, the modified writer will convert it to
// pagefile backing when it notices the change later.
//
MI_SET_PTE_DIRTY (TempPte);
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte);
//
// The TB entry must be flushed as the valid PTE with
// the dirty bit clear has been fetched into the TB. If
// it isn't flushed, another fault is generated as the
// dirty bit is not set in the cached TB entry.
//
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
return FALSE;
}
break;
}
NextEntry = NextEntry->Flink;
}
}
}
else {
WorkingSetList = MmWorkingSetList;
SessionSpace = NULL;
//
// If a fork operation is in progress, block until the fork is
// completed, then retry the whole operation as the state of
// everything may have changed between when the mutexes were
// released and reacquired.
//
if (CurrentProcess->ForkInProgress != NULL) {
if (MiWaitForForkToComplete (CurrentProcess) == TRUE) {
return FALSE;
}
}
if (TempPte.u.Hard.CopyOnWrite == 0) {
//
// This is a fork page which is being made private in order
// to change the protection of the page.
// Do not make the page writable.
//
FakeCopyOnWrite = TRUE;
}
}
WorkingSetIndex = MiLocateWsle (FaultingAddress,
WorkingSetList,
Pfn1->u1.WsIndex,
FALSE);
//
// The page must be copied into a new page.
//
LOCK_PFN (OldIrql);
if ((MmAvailablePages < MM_HIGH_LIMIT) &&
(MiEnsureAvailablePageOrWait (SessionSpace != NULL ? HYDRA_PROCESS : CurrentProcess, OldIrql))) {
//
// A wait operation was performed to obtain an available
// page and the working set mutex and PFN lock have
// been released and various things may have changed for
// the worse. Rather than examine all the conditions again,
// return and if things are still proper, the fault will
// be taken again.
//
UNLOCK_PFN (OldIrql);
return FALSE;
}
//
// This must be a prototype PTE. Perform the copy on write.
//
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
//
// A page is being copied and made private, the global state of
// the shared page needs to be updated at this point on certain
// hardware. This is done by ORing the dirty bit into the modify bit in
// the PFN element.
//
// Note that a session page cannot be dirty (no POSIX-style forking is
// supported for these drivers).
//
if (SessionSpace != NULL) {
ASSERT ((TempPte.u.Hard.Valid == 1) && (TempPte.u.Hard.Write == 0));
ASSERT (!MI_IS_PTE_DIRTY (TempPte));
NewPageIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_SESSION(MmSessionSpace));
}
else {
MI_CAPTURE_DIRTY_BIT_TO_PFN (PointerPte, Pfn1);
CloneBlock = (PMMCLONE_BLOCK) Pfn1->PteAddress;
//
// Get a new page with the same color as this page.
//
NewPageIndex = MiRemoveAnyPage (
MI_PAGE_COLOR_PTE_PROCESS(PageFrameIndex,
&CurrentProcess->NextPageColor));
}
MiInitializeCopyOnWritePfn (NewPageIndex,
PointerPte,
WorkingSetIndex,
WorkingSetList);
UNLOCK_PFN (OldIrql);
InterlockedIncrement (&KeGetCurrentPrcb ()->MmCopyOnWriteCount);
CopyFrom = PAGE_ALIGN (FaultingAddress);
MappingPte = MiReserveSystemPtes (1, SystemPteSpace);
if (MappingPte != NULL) {
MI_MAKE_VALID_KERNEL_PTE (TempPte2,
NewPageIndex,
MM_READWRITE,
MappingPte);
MI_SET_PTE_DIRTY (TempPte2);
if (Pfn1->u3.e1.CacheAttribute == MiNonCached) {
MI_DISABLE_CACHING (TempPte2);
}
else if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined) {
MI_SET_PTE_WRITE_COMBINE (TempPte2);
}
MI_WRITE_VALID_PTE (MappingPte, TempPte2);
CopyTo = MiGetVirtualAddressMappedByPte (MappingPte);
}
else {
CopyTo = MiMapPageInHyperSpace (CurrentProcess,
NewPageIndex,
&OldIrql);
}
KeCopyPage (CopyTo, CopyFrom);
if (MappingPte != NULL) {
MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace);
}
else {
MiUnmapPageInHyperSpace (CurrentProcess, CopyTo, OldIrql);
}
if (!FakeCopyOnWrite) {
//
// If the page was really a copy on write page, make it
// accessed, dirty and writable. Also, clear the copy-on-write
// bit in the PTE.
//
MI_SET_PTE_DIRTY (TempPte);
TempPte.u.Hard.Write = 1;
MI_SET_ACCESSED_IN_PTE (&TempPte, 1);
TempPte.u.Hard.CopyOnWrite = 0;
}
//
// Regardless of whether the page was really a copy on write,
// the frame field of the PTE must be updated.
//
TempPte.u.Hard.PageFrameNumber = NewPageIndex;
//
// If the modify bit is set in the PFN database for the
// page, the data cache must be flushed. This is due to the
// fact that this process may have been cloned and the cache
// still contains stale data destined for the page we are
// going to remove.
//
ASSERT (TempPte.u.Hard.Valid == 1);
MI_WRITE_VALID_PTE_NEW_PAGE (PointerPte, TempPte);
//
// Flush the TB entry for this page.
//
if (SessionSpace == NULL) {
MI_FLUSH_SINGLE_TB (FaultingAddress, FALSE);
//
// Increment the number of private pages.
//
CurrentProcess->NumberOfPrivatePages += 1;
}
else {
MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE);
ASSERT (Pfn1->u3.e1.PrototypePte == 1);
}
//
// Decrement the share count for the page which was copied
// as this PTE no longer refers to it.
//
LOCK_PFN (OldIrql);
MiDecrementShareCount (Pfn1, PageFrameIndex);
if (SessionSpace == NULL) {
CloneDescriptor = MiLocateCloneAddress (CurrentProcess,
(PVOID)CloneBlock);
if (CloneDescriptor != NULL) {
//
// Decrement the reference count for the clone block,
// note that this could release and reacquire the mutexes.
//
MiDecrementCloneBlockReference (CloneDescriptor,
CloneBlock,
CurrentProcess,
NULL,
OldIrql);
}
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
