NTSTATUS
NTAPI
MmNotPresentFaultCacheSection(KPROCESSOR_MODE Mode,
ULONG_PTR Address,
BOOLEAN FromMdl)
{
PETHREAD Thread;
PMMSUPPORT AddressSpace;
NTSTATUS Status;
Address &= ~(PAGE_SIZE - 1);
DPRINT("MmNotPresentFault(Mode %d, Address %Ix)\n", Mode, Address);
Thread = PsGetCurrentThread();
if (KeGetCurrentIrql() >= DISPATCH_LEVEL)
{
DPRINT1("Page fault at high IRQL %u, address %Ix\n",
KeGetCurrentIrql(),
Address);
ASSERT(FALSE);
return STATUS_UNSUCCESSFUL;
}
/* Find the memory area for the faulting address */
if (Address >= (ULONG_PTR)MmSystemRangeStart)
{
/* Check permissions */
if (Mode != KernelMode)
{
DPRINTC("Address: %x\n", Address);
return STATUS_ACCESS_VIOLATION;
}
AddressSpace = MmGetKernelAddressSpace();
}
else
{
AddressSpace = &PsGetCurrentProcess()->Vm;
}
Thread->ActiveFaultCount++;
Status = MmNotPresentFaultCacheSectionInner(Mode,
AddressSpace,
Address,
FromMdl,
Thread);
Thread->ActiveFaultCount--;
ASSERT(Status != STATUS_UNSUCCESSFUL);
ASSERT(Status != STATUS_INVALID_PARAMETER);
DPRINT("MmAccessFault %p:%Ix -> %x\n",
MmGetAddressSpaceOwner(AddressSpace),
Address,
Status);
return Status;
}
//
// Helper function to create initial memory areas.
// The created area is always read/write.
//
VOID
INIT_FUNCTION
NTAPI
MiCreateArm3StaticMemoryArea(PVOID BaseAddress, ULONG Size, BOOLEAN Executable)
{
const ULONG Protection = Executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
PVOID pBaseAddress = BaseAddress;
PMEMORY_AREA MArea;
NTSTATUS Status;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&pBaseAddress,
Size,
Protection,
&MArea,
0,
PAGE_SIZE);
ASSERT(Status == STATUS_SUCCESS);
// TODO: Perhaps it would be prudent to bugcheck here, not only assert?
}
VOID
INIT_FUNCTION
NTAPI
MiInitSystemMemoryAreas()
{
PVOID BaseAddress;
PHYSICAL_ADDRESS BoundaryAddressMultiple;
PMEMORY_AREA MArea;
NTSTATUS Status;
BoundaryAddressMultiple.QuadPart = 0;
//
// Create the memory area to define the PTE base
//
BaseAddress = (PVOID)PTE_BASE;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
4 * 1024 * 1024,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// Create the memory area to define Hyperspace
//
BaseAddress = (PVOID)HYPER_SPACE;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
4 * 1024 * 1024,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// Protect the PFN database
//
BaseAddress = MmPfnDatabase;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
(MxPfnAllocation << PAGE_SHIFT),
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// Odyssey requires a memory area to keep the initial NP area off-bounds
//
BaseAddress = MmNonPagedPoolStart;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
MmSizeOfNonPagedPoolInBytes,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// And we need one more for the system NP
//
BaseAddress = MmNonPagedSystemStart;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
(ULONG_PTR)MmNonPagedPoolEnd -
(ULONG_PTR)MmNonPagedSystemStart,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// We also need one for system view space
//
BaseAddress = MiSystemViewStart;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
MmSystemViewSize,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// And another for session space
//
BaseAddress = MmSessionBase;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
(ULONG_PTR)MiSessionSpaceEnd -
(ULONG_PTR)MmSessionBase,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// One more for ARM paged pool
//
BaseAddress = MmPagedPoolStart;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
MmSizeOfPagedPoolInBytes,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// Next, the KPCR
//
BaseAddress = (PVOID)PCR;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
PAGE_SIZE * KeNumberProcessors,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// Now the KUSER_SHARED_DATA
//
BaseAddress = (PVOID)KI_USER_SHARED_DATA;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
PAGE_SIZE,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
//
// And the debugger mapping
//
BaseAddress = MI_DEBUG_MAPPING;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
PAGE_SIZE,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
#if defined(_X86_)
//
// Finally, reserve the 2 pages we currently make use of for HAL mappings
//
BaseAddress = (PVOID)0xFFC00000;
Status = MmCreateMemoryArea(MmGetKernelAddressSpace(),
MEMORY_AREA_OWNED_BY_ARM3 | MEMORY_AREA_STATIC,
&BaseAddress,
PAGE_SIZE * 2,
PAGE_READWRITE,
&MArea,
TRUE,
0,
BoundaryAddressMultiple);
ASSERT(Status == STATUS_SUCCESS);
#endif
}
NTSTATUS
NTAPI
MmpSectionAccessFaultInner(KPROCESSOR_MODE Mode,
PMMSUPPORT AddressSpace,
ULONG_PTR Address,
BOOLEAN FromMdl,
PETHREAD Thread)
{
MEMORY_AREA* MemoryArea;
NTSTATUS Status;
BOOLEAN Locked = FromMdl;
MM_REQUIRED_RESOURCES Resources = { 0 };
WORK_QUEUE_WITH_CONTEXT Context;
RtlZeroMemory(&Context, sizeof(WORK_QUEUE_WITH_CONTEXT));
DPRINT("MmAccessFault(Mode %d, Address %Ix)\n", Mode, Address);
if (KeGetCurrentIrql() >= DISPATCH_LEVEL)
{
DPRINT1("Page fault at high IRQL was %u\n", KeGetCurrentIrql());
return STATUS_UNSUCCESSFUL;
}
/* Find the memory area for the faulting address */
if (Address >= (ULONG_PTR)MmSystemRangeStart)
{
/* Check permissions */
if (Mode != KernelMode)
{
DPRINT("MmAccessFault(Mode %d, Address %Ix)\n", Mode, Address);
return STATUS_ACCESS_VIOLATION;
}
AddressSpace = MmGetKernelAddressSpace();
}
else
{
AddressSpace = &PsGetCurrentProcess()->Vm;
}
if (!FromMdl)
{
MmLockAddressSpace(AddressSpace);
}
do
{
MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, (PVOID)Address);
if (MemoryArea == NULL ||
MemoryArea->DeleteInProgress)
{
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
DPRINT("Address: %Ix\n", Address);
return STATUS_ACCESS_VIOLATION;
}
DPRINT("Type %x (%p -> %p)\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
Resources.DoAcquisition = NULL;
// Note: fault handlers are called with address space locked
// We return STATUS_MORE_PROCESSING_REQUIRED if anything is needed
Status = MiCowCacheSectionPage(AddressSpace,
MemoryArea,
(PVOID)Address,
Locked,
&Resources);
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
if (Status == STATUS_SUCCESS + 1)
{
/* Wait page ... */
DPRINT("Waiting for %Ix\n", Address);
MiWaitForPageEvent(MmGetAddressSpaceOwner(AddressSpace), Address);
DPRINT("Restarting fault %Ix\n", Address);
Status = STATUS_MM_RESTART_OPERATION;
}
else if (Status == STATUS_MM_RESTART_OPERATION)
{
/* Clean slate */
RtlZeroMemory(&Resources, sizeof(Resources));
}
else if (Status == STATUS_MORE_PROCESSING_REQUIRED)
{
if (Thread->ActiveFaultCount > 0)
{
DPRINT("Already fault handling ... going to work item (%Ix)\n",
Address);
Context.AddressSpace = AddressSpace;
Context.MemoryArea = MemoryArea;
Context.Required = &Resources;
KeInitializeEvent(&Context.Wait, NotificationEvent, FALSE);
ExInitializeWorkItem(&Context.WorkItem,
MmpFaultWorker,
&Context);
DPRINT("Queue work item\n");
ExQueueWorkItem(&Context.WorkItem, DelayedWorkQueue);
DPRINT("Wait\n");
KeWaitForSingleObject(&Context.Wait, 0, KernelMode, FALSE, NULL);
Status = Context.Status;
DPRINT("Status %x\n", Status);
}
else
{
Status = Resources.DoAcquisition(AddressSpace, MemoryArea, &Resources);
}
if (NT_SUCCESS(Status))
{
Status = STATUS_MM_RESTART_OPERATION;
}
}
if (!FromMdl)
{
MmLockAddressSpace(AddressSpace);
}
}
while (Status == STATUS_MM_RESTART_OPERATION);
if (!NT_SUCCESS(Status) && MemoryArea->Type == 1)
{
DPRINT1("Completed page fault handling %Ix %x\n", Address, Status);
DPRINT1("Type %x (%p -> %p)\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
}
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
return Status;
}
NTSTATUS
NTAPI
MmpPageOutPhysicalAddress(PFN_NUMBER Page)
{
BOOLEAN ProcRef = FALSE, PageDirty;
PFN_NUMBER SectionPage = 0;
PMM_RMAP_ENTRY entry;
PMM_SECTION_SEGMENT Segment = NULL;
LARGE_INTEGER FileOffset;
PMEMORY_AREA MemoryArea;
PMMSUPPORT AddressSpace = NULL;
BOOLEAN Dirty = FALSE;
PVOID Address = NULL;
PEPROCESS Process = NULL;
NTSTATUS Status = STATUS_SUCCESS;
MM_REQUIRED_RESOURCES Resources = { 0 };
DPRINTC("Page out %x (ref ct %x)\n", Page, MmGetReferenceCountPage(Page));
ExAcquireFastMutex(&MiGlobalPageOperation);
if ((Segment = MmGetSectionAssociation(Page, &FileOffset)))
{
DPRINTC("Withdrawing page (%x) %p:%x\n",
Page,
Segment,
FileOffset.LowPart);
SectionPage = MmWithdrawSectionPage(Segment, &FileOffset, &Dirty);
DPRINTC("SectionPage %x\n", SectionPage);
if (SectionPage == MM_WAIT_ENTRY || SectionPage == 0)
{
DPRINT1("In progress page out %x\n", SectionPage);
ExReleaseFastMutex(&MiGlobalPageOperation);
return STATUS_UNSUCCESSFUL;
}
else
{
ASSERT(SectionPage == Page);
}
Resources.State = Dirty ? 1 : 0;
}
else
{
DPRINT("No segment association for %x\n", Page);
}
Dirty = MmIsDirtyPageRmap(Page);
DPRINTC("Trying to unmap all instances of %x\n", Page);
ExAcquireFastMutex(&RmapListLock);
entry = MmGetRmapListHeadPage(Page);
// Entry and Segment might be null here in the case that the page
// is new and is in the process of being swapped in
if (!entry && !Segment)
{
Status = STATUS_UNSUCCESSFUL;
DPRINT1("Page %x is in transit\n", Page);
ExReleaseFastMutex(&RmapListLock);
goto bail;
}
while (entry != NULL && NT_SUCCESS(Status))
{
Process = entry->Process;
Address = entry->Address;
DPRINTC("Process %p Address %p Page %x\n", Process, Address, Page);
if (RMAP_IS_SEGMENT(Address))
{
entry = entry->Next;
continue;
}
if (Process && Address < MmSystemRangeStart)
{
/* Make sure we don't try to page out part of an exiting process */
if (PspIsProcessExiting(Process))
{
DPRINT("bail\n");
ExReleaseFastMutex(&RmapListLock);
goto bail;
}
ObReferenceObject(Process);
ProcRef = TRUE;
AddressSpace = &Process->Vm;
}
else
{
AddressSpace = MmGetKernelAddressSpace();
}
ExReleaseFastMutex(&RmapListLock);
RtlZeroMemory(&Resources, sizeof(Resources));
if ((((ULONG_PTR)Address) & 0xFFF) != 0)
{
KeBugCheck(MEMORY_MANAGEMENT);
}
do
{
MmLockAddressSpace(AddressSpace);
MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
{
Status = STATUS_UNSUCCESSFUL;
MmUnlockAddressSpace(AddressSpace);
DPRINTC("bail\n");
goto bail;
}
DPRINTC("Type %x (%p -> %p)\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
Resources.DoAcquisition = NULL;
Resources.Page[0] = Page;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
DPRINT("%p:%p, page %x %x\n",
Process,
Address,
Page,
Resources.Page[0]);
PageDirty = FALSE;
Status = MmPageOutCacheSection(AddressSpace,
MemoryArea,
Address,
&PageDirty,
&Resources);
Dirty |= PageDirty;
DPRINT("%x\n", Status);
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
MmUnlockAddressSpace(AddressSpace);
if (Status == STATUS_SUCCESS + 1)
{
// Wait page ... the other guy has it, so we'll just fail for now
DPRINT1("Wait entry ... can't continue\n");
Status = STATUS_UNSUCCESSFUL;
goto bail;
}
else if (Status == STATUS_MORE_PROCESSING_REQUIRED)
{
DPRINTC("DoAcquisition %p\n", Resources.DoAcquisition);
Status = Resources.DoAcquisition(AddressSpace,
MemoryArea,
&Resources);
DPRINTC("Status %x\n", Status);
if (!NT_SUCCESS(Status))
{
DPRINT1("bail\n");
goto bail;
}
else
{
Status = STATUS_MM_RESTART_OPERATION;
}
}
}
while (Status == STATUS_MM_RESTART_OPERATION);
if (ProcRef)
{
ObDereferenceObject(Process);
ProcRef = FALSE;
}
ExAcquireFastMutex(&RmapListLock);
ASSERT(!MM_IS_WAIT_PTE(MmGetPfnForProcess(Process, Address)));
entry = MmGetRmapListHeadPage(Page);
DPRINTC("Entry %p\n", entry);
}
ExReleaseFastMutex(&RmapListLock);
bail:
DPRINTC("BAIL %x\n", Status);
if (Segment)
{
ULONG RefCount;
DPRINTC("About to finalize section page %x (%p:%x) Status %x %s\n",
Page,
Segment,
FileOffset.LowPart,
Status,
Dirty ? "dirty" : "clean");
if (!NT_SUCCESS(Status) ||
!NT_SUCCESS(Status = MmFinalizeSectionPageOut(Segment,
&FileOffset,
Page,
Dirty)))
{
DPRINTC("Failed to page out %x, replacing %x at %x in segment %x\n",
SectionPage,
FileOffset.LowPart,
Segment);
MmLockSectionSegment(Segment);
MmSetPageEntrySectionSegment(Segment,
&FileOffset,
Dirty ? MAKE_PFN_SSE(Page) : DIRTY_SSE(MAKE_PFN_SSE(Page)));
MmUnlockSectionSegment(Segment);
}
/* Alas, we had the last reference */
if ((RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0)
MmFinalizeSegment(Segment);
}
if (ProcRef)
{
DPRINTC("Dereferencing process...\n");
ObDereferenceObject(Process);
}
ExReleaseFastMutex(&MiGlobalPageOperation);
DPRINTC("%s %x %x\n",
NT_SUCCESS(Status) ? "Evicted" : "Spared",
Page,
Status);
return NT_SUCCESS(Status) ? STATUS_SUCCESS : STATUS_UNSUCCESSFUL;
}
