static VOID
MmInsertMemoryArea(
PMMSUPPORT AddressSpace,
PMEMORY_AREA marea)
{
PMEMORY_AREA Node;
PMEMORY_AREA PreviousNode;
ULONG Depth = 0;
PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
/* Build a lame VAD if this is a user-space allocation */
if ((marea->EndingAddress < MmSystemRangeStart) && (marea->Type != MEMORY_AREA_OWNED_BY_ARM3))
{
PMMVAD Vad;
ASSERT(marea->Type == MEMORY_AREA_SECTION_VIEW || marea->Type == MEMORY_AREA_CACHE);
Vad = ExAllocatePoolWithTag(NonPagedPool, sizeof(MMVAD), TAG_MVAD);
ASSERT(Vad);
RtlZeroMemory(Vad, sizeof(MMVAD));
Vad->StartingVpn = PAGE_ROUND_DOWN(marea->StartingAddress) >> PAGE_SHIFT;
/*
* For some strange reason, it is perfectly valid to create a MAREA from 0x1000 to... 0x1000.
* In a normal OS/Memory Manager, this would be retarded, but ReactOS allows this (how it works
* I don't even want to know).
*/
if (marea->EndingAddress != marea->StartingAddress)
{
Vad->EndingVpn = PAGE_ROUND_DOWN((ULONG_PTR)marea->EndingAddress - 1) >> PAGE_SHIFT;
}
NTSTATUS
NTAPI
MmNotPresentFaultCacheSection(KPROCESSOR_MODE Mode,
ULONG_PTR Address,
BOOLEAN FromMdl)
{
PETHREAD Thread;
PMMSUPPORT AddressSpace;
NTSTATUS Status;
Address &= ~(PAGE_SIZE - 1);
DPRINT("MmNotPresentFault(Mode %d, Address %Ix)\n", Mode, Address);
Thread = PsGetCurrentThread();
if (KeGetCurrentIrql() >= DISPATCH_LEVEL)
{
DPRINT1("Page fault at high IRQL %u, address %Ix\n",
KeGetCurrentIrql(),
Address);
ASSERT(FALSE);
return STATUS_UNSUCCESSFUL;
}
/* Find the memory area for the faulting address */
if (Address >= (ULONG_PTR)MmSystemRangeStart)
{
/* Check permissions */
if (Mode != KernelMode)
{
DPRINTC("Address: %x\n", Address);
return STATUS_ACCESS_VIOLATION;
}
AddressSpace = MmGetKernelAddressSpace();
}
else
{
AddressSpace = &PsGetCurrentProcess()->Vm;
}
Thread->ActiveFaultCount++;
Status = MmNotPresentFaultCacheSectionInner(Mode,
AddressSpace,
Address,
FromMdl,
Thread);
Thread->ActiveFaultCount--;
ASSERT(Status != STATUS_UNSUCCESSFUL);
ASSERT(Status != STATUS_INVALID_PARAMETER);
DPRINT("MmAccessFault %p:%Ix -> %x\n",
MmGetAddressSpaceOwner(AddressSpace),
Address,
Status);
return Status;
}
NTSTATUS
NTAPI
MmPageOutCacheSection(PMMSUPPORT AddressSpace,
MEMORY_AREA* MemoryArea,
PVOID Address,
PBOOLEAN Dirty,
PMM_REQUIRED_RESOURCES Required)
{
ULONG_PTR Entry;
PFN_NUMBER OurPage;
PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
LARGE_INTEGER TotalOffset;
PMM_SECTION_SEGMENT Segment;
PVOID PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE);
TotalOffset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress +
MemoryArea->Data.SectionData.ViewOffset.QuadPart;
Segment = MemoryArea->Data.SectionData.Segment;
MmLockSectionSegment(Segment);
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset);
DBG_UNREFERENCED_LOCAL_VARIABLE(Entry);
if (MmIsPageSwapEntry(Process, PAddress))
{
SWAPENTRY SwapEntry;
MmGetPageFileMapping(Process, PAddress, &SwapEntry);
MmUnlockSectionSegment(Segment);
return SwapEntry == MM_WAIT_ENTRY ? STATUS_SUCCESS + 1 : STATUS_UNSUCCESSFUL;
}
MmDeleteRmap(Required->Page[0], Process, Address);
MmDeleteVirtualMapping(Process, Address, Dirty, &OurPage);
ASSERT(OurPage == Required->Page[0]);
/* Note: this releases the reference held by this address space only. */
MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]);
MmUnlockSectionSegment(Segment);
MiSetPageEvent(Process, Address);
return STATUS_SUCCESS;
}
NTSTATUS
NTAPI
MmNotPresentFaultCacheSectionInner(KPROCESSOR_MODE Mode,
PMMSUPPORT AddressSpace,
ULONG_PTR Address,
BOOLEAN FromMdl,
PETHREAD Thread)
{
BOOLEAN Locked = FromMdl;
PMEMORY_AREA MemoryArea;
MM_REQUIRED_RESOURCES Resources = { 0 };
WORK_QUEUE_WITH_CONTEXT Context;
NTSTATUS Status = STATUS_SUCCESS;
RtlZeroMemory(&Context, sizeof(WORK_QUEUE_WITH_CONTEXT));
if (!FromMdl)
{
MmLockAddressSpace(AddressSpace);
}
/* Call the memory area specific fault handler */
do
{
MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, (PVOID)Address);
if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
{
Status = STATUS_ACCESS_VIOLATION;
if (MemoryArea)
{
DPRINT1("Type %x DIP %x\n",
MemoryArea->Type,
MemoryArea->DeleteInProgress);
}
else
{
DPRINT1("No memory area\n");
}
DPRINT1("Process %p, Address %Ix\n",
MmGetAddressSpaceOwner(AddressSpace),
Address);
break;
}
DPRINTC("Type %x (%p -> %08Ix -> %p) in %p\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
Address,
MemoryArea->EndingAddress,
PsGetCurrentThread());
Resources.DoAcquisition = NULL;
// Note: fault handlers are called with address space locked
// We return STATUS_MORE_PROCESSING_REQUIRED if anything is needed
Status = MmNotPresentFaultCachePage(AddressSpace,
MemoryArea,
(PVOID)Address,
Locked,
&Resources);
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
if (Status == STATUS_SUCCESS)
{
; // Nothing
}
else if (Status == STATUS_SUCCESS + 1)
{
/* Wait page ... */
DPRINT("Waiting for %Ix\n", Address);
MiWaitForPageEvent(MmGetAddressSpaceOwner(AddressSpace), Address);
DPRINT("Done waiting for %Ix\n", Address);
Status = STATUS_MM_RESTART_OPERATION;
}
else if (Status == STATUS_MM_RESTART_OPERATION)
{
/* Clean slate */
DPRINT("Clear resource\n");
RtlZeroMemory(&Resources, sizeof(Resources));
}
else if (Status == STATUS_MORE_PROCESSING_REQUIRED)
{
if (Thread->ActiveFaultCount > 2)
{
DPRINTC("Already fault handling ... going to work item (%Ix)\n", Address);
Context.AddressSpace = AddressSpace;
Context.MemoryArea = MemoryArea;
Context.Required = &Resources;
KeInitializeEvent(&Context.Wait, NotificationEvent, FALSE);
ExInitializeWorkItem(&Context.WorkItem,
(PWORKER_THREAD_ROUTINE)MmpFaultWorker,
&Context);
DPRINT("Queue work item\n");
ExQueueWorkItem(&Context.WorkItem, DelayedWorkQueue);
DPRINT("Wait\n");
KeWaitForSingleObject(&Context.Wait, 0, KernelMode, FALSE, NULL);
Status = Context.Status;
DPRINTC("Status %x\n", Status);
}
else
{
DPRINT("DoAcquisition %p\n", Resources.DoAcquisition);
Status = Resources.DoAcquisition(AddressSpace,
MemoryArea,
&Resources);
DPRINT("DoAcquisition %p -> %x\n",
Resources.DoAcquisition,
Status);
}
if (NT_SUCCESS(Status))
{
Status = STATUS_MM_RESTART_OPERATION;
}
}
else if (NT_SUCCESS(Status))
{
ASSERT(FALSE);
}
if (!FromMdl)
{
MmLockAddressSpace(AddressSpace);
}
}
while (Status == STATUS_MM_RESTART_OPERATION);
DPRINTC("Completed page fault handling: %p:%Ix %x\n",
MmGetAddressSpaceOwner(AddressSpace),
Address,
Status);
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
MiSetPageEvent(MmGetAddressSpaceOwner(AddressSpace), Address);
DPRINT("Done %x\n", Status);
return Status;
}
NTSTATUS
NTAPI
MmpSectionAccessFaultInner(KPROCESSOR_MODE Mode,
PMMSUPPORT AddressSpace,
ULONG_PTR Address,
BOOLEAN FromMdl,
PETHREAD Thread)
{
MEMORY_AREA* MemoryArea;
NTSTATUS Status;
BOOLEAN Locked = FromMdl;
MM_REQUIRED_RESOURCES Resources = { 0 };
WORK_QUEUE_WITH_CONTEXT Context;
RtlZeroMemory(&Context, sizeof(WORK_QUEUE_WITH_CONTEXT));
DPRINT("MmAccessFault(Mode %d, Address %Ix)\n", Mode, Address);
if (KeGetCurrentIrql() >= DISPATCH_LEVEL)
{
DPRINT1("Page fault at high IRQL was %u\n", KeGetCurrentIrql());
return STATUS_UNSUCCESSFUL;
}
/* Find the memory area for the faulting address */
if (Address >= (ULONG_PTR)MmSystemRangeStart)
{
/* Check permissions */
if (Mode != KernelMode)
{
DPRINT("MmAccessFault(Mode %d, Address %Ix)\n", Mode, Address);
return STATUS_ACCESS_VIOLATION;
}
AddressSpace = MmGetKernelAddressSpace();
}
else
{
AddressSpace = &PsGetCurrentProcess()->Vm;
}
if (!FromMdl)
{
MmLockAddressSpace(AddressSpace);
}
do
{
MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, (PVOID)Address);
if (MemoryArea == NULL ||
MemoryArea->DeleteInProgress)
{
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
DPRINT("Address: %Ix\n", Address);
return STATUS_ACCESS_VIOLATION;
}
DPRINT("Type %x (%p -> %p)\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
Resources.DoAcquisition = NULL;
// Note: fault handlers are called with address space locked
// We return STATUS_MORE_PROCESSING_REQUIRED if anything is needed
Status = MiCowCacheSectionPage(AddressSpace,
MemoryArea,
(PVOID)Address,
Locked,
&Resources);
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
if (Status == STATUS_SUCCESS + 1)
{
/* Wait page ... */
DPRINT("Waiting for %Ix\n", Address);
MiWaitForPageEvent(MmGetAddressSpaceOwner(AddressSpace), Address);
DPRINT("Restarting fault %Ix\n", Address);
Status = STATUS_MM_RESTART_OPERATION;
}
else if (Status == STATUS_MM_RESTART_OPERATION)
{
/* Clean slate */
RtlZeroMemory(&Resources, sizeof(Resources));
}
else if (Status == STATUS_MORE_PROCESSING_REQUIRED)
{
if (Thread->ActiveFaultCount > 0)
{
DPRINT("Already fault handling ... going to work item (%Ix)\n",
Address);
Context.AddressSpace = AddressSpace;
Context.MemoryArea = MemoryArea;
Context.Required = &Resources;
KeInitializeEvent(&Context.Wait, NotificationEvent, FALSE);
ExInitializeWorkItem(&Context.WorkItem,
MmpFaultWorker,
&Context);
DPRINT("Queue work item\n");
ExQueueWorkItem(&Context.WorkItem, DelayedWorkQueue);
DPRINT("Wait\n");
KeWaitForSingleObject(&Context.Wait, 0, KernelMode, FALSE, NULL);
Status = Context.Status;
DPRINT("Status %x\n", Status);
}
else
{
Status = Resources.DoAcquisition(AddressSpace, MemoryArea, &Resources);
}
if (NT_SUCCESS(Status))
{
Status = STATUS_MM_RESTART_OPERATION;
}
}
if (!FromMdl)
{
MmLockAddressSpace(AddressSpace);
}
}
while (Status == STATUS_MM_RESTART_OPERATION);
if (!NT_SUCCESS(Status) && MemoryArea->Type == 1)
{
DPRINT1("Completed page fault handling %Ix %x\n", Address, Status);
DPRINT1("Type %x (%p -> %p)\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
}
if (!FromMdl)
{
MmUnlockAddressSpace(AddressSpace);
}
return Status;
}
NTSTATUS
NTAPI
MiCowCacheSectionPage (
_In_ PMMSUPPORT AddressSpace,
_In_ PMEMORY_AREA MemoryArea,
_In_ PVOID Address,
_In_ BOOLEAN Locked,
_Inout_ PMM_REQUIRED_RESOURCES Required)
{
PMM_SECTION_SEGMENT Segment;
PFN_NUMBER NewPage, OldPage;
NTSTATUS Status;
PVOID PAddress;
LARGE_INTEGER Offset;
PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
DPRINT("MmAccessFaultSectionView(%p, %p, %p, %u)\n",
AddressSpace,
MemoryArea,
Address,
Locked);
Segment = MemoryArea->Data.SectionData.Segment;
/* Lock the segment */
MmLockSectionSegment(Segment);
/* Find the offset of the page */
PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE);
Offset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress +
MemoryArea->Data.SectionData.ViewOffset.QuadPart;
if (!Segment->WriteCopy /*&&
!MemoryArea->Data.SectionData.WriteCopyView*/ ||
Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)
{
#if 0
if (Region->Protect == PAGE_READWRITE ||
Region->Protect == PAGE_EXECUTE_READWRITE)
#endif
{
ULONG_PTR Entry;
DPRINTC("setting non-cow page %p %p:%p offset %I64x (%Ix) to writable\n",
Segment,
Process,
PAddress,
Offset.QuadPart,
MmGetPfnForProcess(Process, Address));
if (Segment->FileObject)
{
DPRINTC("file %wZ\n", &Segment->FileObject->FileName);
}
Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
DPRINT("Entry %x\n", Entry);
if (Entry &&
!IS_SWAP_FROM_SSE(Entry) &&
PFN_FROM_SSE(Entry) == MmGetPfnForProcess(Process, Address)) {
MmSetPageEntrySectionSegment(Segment,
&Offset,
DIRTY_SSE(Entry));
}
MmSetPageProtect(Process, PAddress, PAGE_READWRITE);
MmSetDirtyPage(Process, PAddress);
MmUnlockSectionSegment(Segment);
DPRINT("Done\n");
return STATUS_SUCCESS;
}
#if 0
else
{
DPRINT("Not supposed to be writable\n");
MmUnlockSectionSegment(Segment);
return STATUS_ACCESS_VIOLATION;
}
#endif
}
if (!Required->Page[0])
{
SWAPENTRY SwapEntry;
if (MmIsPageSwapEntry(Process, Address))
{
MmGetPageFileMapping(Process, Address, &SwapEntry);
MmUnlockSectionSegment(Segment);
if (SwapEntry == MM_WAIT_ENTRY)
return STATUS_SUCCESS + 1; // Wait ... somebody else is getting it right now
else
return STATUS_SUCCESS; // Nonwait swap entry ... handle elsewhere
}
/* Call out to acquire a page to copy to. We'll be re-called when
* the page has been allocated. */
Required->Page[1] = MmGetPfnForProcess(Process, Address);
Required->Consumer = MC_CACHE;
Required->Amount = 1;
Required->File = __FILE__;
Required->Line = __LINE__;
Required->DoAcquisition = MiGetOnePage;
MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY);
MmUnlockSectionSegment(Segment);
return STATUS_MORE_PROCESSING_REQUIRED;
}
NewPage = Required->Page[0];
OldPage = Required->Page[1];
DPRINT("Allocated page %x\n", NewPage);
/* Unshare the old page */
MmDeleteRmap(OldPage, Process, PAddress);
/* Copy the old page */
DPRINT("Copying\n");
MiCopyPageToPage(NewPage, OldPage);
/* Set the PTE to point to the new page */
Status = MmCreateVirtualMapping(Process,
Address,
PAGE_READWRITE,
&NewPage,
1);
if (!NT_SUCCESS(Status))
{
DPRINT1("MmCreateVirtualMapping failed, not out of memory\n");
ASSERT(FALSE);
MmUnlockSectionSegment(Segment);
return Status;
}
MmInsertRmap(NewPage, Process, PAddress);
MmReleasePageMemoryConsumer(MC_CACHE, OldPage);
MmUnlockSectionSegment(Segment);
DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS;
}
NTSTATUS
NTAPI
MmNotPresentFaultCachePage (
_In_ PMMSUPPORT AddressSpace,
_In_ MEMORY_AREA* MemoryArea,
_In_ PVOID Address,
_In_ BOOLEAN Locked,
_Inout_ PMM_REQUIRED_RESOURCES Required)
{
NTSTATUS Status;
PVOID PAddress;
ULONG Consumer;
PMM_SECTION_SEGMENT Segment;
LARGE_INTEGER FileOffset, TotalOffset;
ULONG_PTR Entry;
ULONG Attributes;
PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
KIRQL OldIrql;
DPRINT("Not Present: %p %p (%p-%p)\n",
AddressSpace,
Address,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
/*
* There is a window between taking the page fault and locking the
* address space when another thread could load the page so we check
* that.
*/
if (MmIsPagePresent(Process, Address))
{
DPRINT("Done\n");
return STATUS_SUCCESS;
}
PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE);
TotalOffset.QuadPart = (ULONG_PTR)PAddress -
(ULONG_PTR)MemoryArea->StartingAddress;
Segment = MemoryArea->Data.SectionData.Segment;
TotalOffset.QuadPart += MemoryArea->Data.SectionData.ViewOffset.QuadPart;
FileOffset = TotalOffset;
//Consumer = (Segment->Flags & MM_DATAFILE_SEGMENT) ? MC_CACHE : MC_USER;
Consumer = MC_CACHE;
if (Segment->FileObject)
{
DPRINT("FileName %wZ\n", &Segment->FileObject->FileName);
}
DPRINT("Total Offset %08x%08x\n", TotalOffset.HighPart, TotalOffset.LowPart);
/* Lock the segment */
MmLockSectionSegment(Segment);
/* Get the entry corresponding to the offset within the section */
Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset);
Attributes = PAGE_READONLY;
if (Required->State && Required->Page[0])
{
DPRINT("Have file and page, set page %x in section @ %x #\n",
Required->Page[0],
TotalOffset.LowPart);
if (Required->SwapEntry)
MmSetSavedSwapEntryPage(Required->Page[0], Required->SwapEntry);
if (Required->State & 2)
{
DPRINT("Set in section @ %x\n", TotalOffset.LowPart);
Status = MmSetPageEntrySectionSegment(Segment,
&TotalOffset,
Entry = MAKE_PFN_SSE(Required->Page[0]));
if (!NT_SUCCESS(Status))
{
MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]);
}
MmUnlockSectionSegment(Segment);
MiSetPageEvent(Process, Address);
DPRINT("Status %x\n", Status);
return STATUS_MM_RESTART_OPERATION;
}
else
{
DPRINT("Set %x in address space @ %p\n", Required->Page[0], Address);
Status = MmCreateVirtualMapping(Process,
Address,
Attributes,
Required->Page,
1);
if (NT_SUCCESS(Status))
{
MmInsertRmap(Required->Page[0], Process, Address);
}
else
{
/* Drop the reference for our address space ... */
MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]);
}
MmUnlockSectionSegment(Segment);
DPRINTC("XXX Set Event %x\n", Status);
MiSetPageEvent(Process, Address);
DPRINT("Status %x\n", Status);
return Status;
}
}
else if (MM_IS_WAIT_PTE(Entry))
{
// Whenever MM_WAIT_ENTRY is required as a swap entry, we need to
// ask the fault handler to wait until we should continue. Rathern
// than recopy this boilerplate code everywhere, we just ask them
// to wait.
MmUnlockSectionSegment(Segment);
return STATUS_SUCCESS + 1;
}
else if (Entry)
{
PFN_NUMBER Page = PFN_FROM_SSE(Entry);
DPRINT("Take reference to page %x #\n", Page);
if (MiGetPfnEntry(Page) == NULL)
{
DPRINT1("Found no PFN entry for page 0x%x in page entry 0x%x (segment: 0x%p, offset: %08x%08x)\n",
Page,
Entry,
Segment,
TotalOffset.HighPart,
TotalOffset.LowPart);
KeBugCheck(CACHE_MANAGER);
}
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
MmReferencePage(Page);
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
Status = MmCreateVirtualMapping(Process, Address, Attributes, &Page, 1);
if (NT_SUCCESS(Status))
{
MmInsertRmap(Page, Process, Address);
}
DPRINT("XXX Set Event %x\n", Status);
MiSetPageEvent(Process, Address);
MmUnlockSectionSegment(Segment);
DPRINT("Status %x\n", Status);
return Status;
}
else
{
DPRINT("Get page into section\n");
/*
* If the entry is zero (and it can't change because we have
* locked the segment) then we need to load the page.
*/
//DPRINT1("Read from file %08x %wZ\n", FileOffset.LowPart, &Section->FileObject->FileName);
Required->State = 2;
Required->Context = Segment->FileObject;
Required->Consumer = Consumer;
Required->FileOffset = FileOffset;
Required->Amount = PAGE_SIZE;
Required->DoAcquisition = MiReadFilePage;
MmSetPageEntrySectionSegment(Segment,
&TotalOffset,
MAKE_SWAP_SSE(MM_WAIT_ENTRY));
MmUnlockSectionSegment(Segment);
return STATUS_MORE_PROCESSING_REQUIRED;
}
ASSERT(FALSE);
return STATUS_ACCESS_VIOLATION;
}
