/* Blocking function to acquire zeroed pages from the balancer. Upon entry: Required->Amount: Number of pages to acquire Required->Consumer: consumer to charge the page to Upon return: Required->Pages[0..Amount]: Allocated pages. The function fails unless all requested pages can be allocated. */ NTSTATUS NTAPI MiGetOnePage(PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PMM_REQUIRED_RESOURCES Required) { ULONG i; NTSTATUS Status = STATUS_SUCCESS; for (i = 0; i < Required->Amount; i++) { DPRINTC("MiGetOnePage(%s:%d)\n", Required->File, Required->Line); Status = MmRequestPageMemoryConsumer(Required->Consumer, TRUE, &Required->Page[i]); if (!NT_SUCCESS(Status)) { while (i > 0) { MmReleasePageMemoryConsumer(Required->Consumer, Required->Page[i-1]); i--; } return Status; } } return Status; }
static VOID MmDeletePageTablePfn(PFN_NUMBER PageFrameNumber, ULONG Level) { PMMPTE PageTable; KIRQL OldIrql; PMMPFN PfnEntry; ULONG i, NumberEntries; /* Check if this is a page table */ if (Level > 0) { NumberEntries = (Level == 4) ? MiAddressToPxi(MmHighestUserAddress)+1 : 512; /* Map the page table in hyperspace */ PageTable = (PMMPTE)MmCreateHyperspaceMapping(PageFrameNumber); /* Loop all page table entries */ for (i = 0; i < NumberEntries; i++) { /* Check if the entry is valid */ if (PageTable[i].u.Hard.Valid) { /* Recursively free the page that backs it */ MmDeletePageTablePfn(PageTable[i].u.Hard.PageFrameNumber, Level - 1); } } /* Delete the hyperspace mapping */ MmDeleteHyperspaceMapping(PageTable); } /* Check if this is a legacy allocation */ PfnEntry = MiGetPfnEntry(PageFrameNumber); if (MI_IS_ROS_PFN(PfnEntry)) { /* Free it using the legacy API */ MmReleasePageMemoryConsumer(MC_SYSTEM, PageFrameNumber); } else { OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Free it using the ARM3 API */ MI_SET_PFN_DELETED(PfnEntry); MiDecrementShareCount(PfnEntry, PageFrameNumber); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } }
NTSTATUS NTAPI MmPageOutCacheSection(PMMSUPPORT AddressSpace, MEMORY_AREA* MemoryArea, PVOID Address, PBOOLEAN Dirty, PMM_REQUIRED_RESOURCES Required) { ULONG_PTR Entry; PFN_NUMBER OurPage; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); LARGE_INTEGER TotalOffset; PMM_SECTION_SEGMENT Segment; PVOID PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); TotalOffset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress + MemoryArea->Data.SectionData.ViewOffset.QuadPart; Segment = MemoryArea->Data.SectionData.Segment; MmLockSectionSegment(Segment); ASSERT(KeGetCurrentIrql() <= APC_LEVEL); Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset); DBG_UNREFERENCED_LOCAL_VARIABLE(Entry); if (MmIsPageSwapEntry(Process, PAddress)) { SWAPENTRY SwapEntry; MmGetPageFileMapping(Process, PAddress, &SwapEntry); MmUnlockSectionSegment(Segment); return SwapEntry == MM_WAIT_ENTRY ? STATUS_SUCCESS + 1 : STATUS_UNSUCCESSFUL; } MmDeleteRmap(Required->Page[0], Process, Address); MmDeleteVirtualMapping(Process, Address, Dirty, &OurPage); ASSERT(OurPage == Required->Page[0]); /* Note: this releases the reference held by this address space only. */ MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); MmUnlockSectionSegment(Segment); MiSetPageEvent(Process, Address); return STATUS_SUCCESS; }
NTSTATUS NTAPI MiCowCacheSectionPage ( _In_ PMMSUPPORT AddressSpace, _In_ PMEMORY_AREA MemoryArea, _In_ PVOID Address, _In_ BOOLEAN Locked, _Inout_ PMM_REQUIRED_RESOURCES Required) { PMM_SECTION_SEGMENT Segment; PFN_NUMBER NewPage, OldPage; NTSTATUS Status; PVOID PAddress; LARGE_INTEGER Offset; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); DPRINT("MmAccessFaultSectionView(%p, %p, %p, %u)\n", AddressSpace, MemoryArea, Address, Locked); Segment = MemoryArea->Data.SectionData.Segment; /* Lock the segment */ MmLockSectionSegment(Segment); /* Find the offset of the page */ PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress + MemoryArea->Data.SectionData.ViewOffset.QuadPart; if (!Segment->WriteCopy /*&& !MemoryArea->Data.SectionData.WriteCopyView*/ || Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { #if 0 if (Region->Protect == PAGE_READWRITE || Region->Protect == PAGE_EXECUTE_READWRITE) #endif { ULONG_PTR Entry; DPRINTC("setting non-cow page %p %p:%p offset %I64x (%Ix) to writable\n", Segment, Process, PAddress, Offset.QuadPart, MmGetPfnForProcess(Process, Address)); if (Segment->FileObject) { DPRINTC("file %wZ\n", &Segment->FileObject->FileName); } Entry = MmGetPageEntrySectionSegment(Segment, &Offset); DPRINT("Entry %x\n", Entry); if (Entry && !IS_SWAP_FROM_SSE(Entry) && PFN_FROM_SSE(Entry) == MmGetPfnForProcess(Process, Address)) { MmSetPageEntrySectionSegment(Segment, &Offset, DIRTY_SSE(Entry)); } MmSetPageProtect(Process, PAddress, PAGE_READWRITE); MmSetDirtyPage(Process, PAddress); MmUnlockSectionSegment(Segment); DPRINT("Done\n"); return STATUS_SUCCESS; } #if 0 else { DPRINT("Not supposed to be writable\n"); MmUnlockSectionSegment(Segment); return STATUS_ACCESS_VIOLATION; } #endif } if (!Required->Page[0]) { SWAPENTRY SwapEntry; if (MmIsPageSwapEntry(Process, Address)) { MmGetPageFileMapping(Process, Address, &SwapEntry); MmUnlockSectionSegment(Segment); if (SwapEntry == MM_WAIT_ENTRY) return STATUS_SUCCESS + 1; // Wait ... somebody else is getting it right now else return STATUS_SUCCESS; // Nonwait swap entry ... handle elsewhere } /* Call out to acquire a page to copy to. We'll be re-called when * the page has been allocated. */ Required->Page[1] = MmGetPfnForProcess(Process, Address); Required->Consumer = MC_CACHE; Required->Amount = 1; Required->File = __FILE__; Required->Line = __LINE__; Required->DoAcquisition = MiGetOnePage; MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY); MmUnlockSectionSegment(Segment); return STATUS_MORE_PROCESSING_REQUIRED; } NewPage = Required->Page[0]; OldPage = Required->Page[1]; DPRINT("Allocated page %x\n", NewPage); /* Unshare the old page */ MmDeleteRmap(OldPage, Process, PAddress); /* Copy the old page */ DPRINT("Copying\n"); MiCopyPageToPage(NewPage, OldPage); /* Set the PTE to point to the new page */ Status = MmCreateVirtualMapping(Process, Address, PAGE_READWRITE, &NewPage, 1); if (!NT_SUCCESS(Status)) { DPRINT1("MmCreateVirtualMapping failed, not out of memory\n"); ASSERT(FALSE); MmUnlockSectionSegment(Segment); return Status; } MmInsertRmap(NewPage, Process, PAddress); MmReleasePageMemoryConsumer(MC_CACHE, OldPage); MmUnlockSectionSegment(Segment); DPRINT("Address 0x%p\n", Address); return STATUS_SUCCESS; }
NTSTATUS NTAPI MmNotPresentFaultCachePage ( _In_ PMMSUPPORT AddressSpace, _In_ MEMORY_AREA* MemoryArea, _In_ PVOID Address, _In_ BOOLEAN Locked, _Inout_ PMM_REQUIRED_RESOURCES Required) { NTSTATUS Status; PVOID PAddress; ULONG Consumer; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER FileOffset, TotalOffset; ULONG_PTR Entry; ULONG Attributes; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); KIRQL OldIrql; DPRINT("Not Present: %p %p (%p-%p)\n", AddressSpace, Address, MemoryArea->StartingAddress, MemoryArea->EndingAddress); /* * There is a window between taking the page fault and locking the * address space when another thread could load the page so we check * that. */ if (MmIsPagePresent(Process, Address)) { DPRINT("Done\n"); return STATUS_SUCCESS; } PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); TotalOffset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress; Segment = MemoryArea->Data.SectionData.Segment; TotalOffset.QuadPart += MemoryArea->Data.SectionData.ViewOffset.QuadPart; FileOffset = TotalOffset; //Consumer = (Segment->Flags & MM_DATAFILE_SEGMENT) ? MC_CACHE : MC_USER; Consumer = MC_CACHE; if (Segment->FileObject) { DPRINT("FileName %wZ\n", &Segment->FileObject->FileName); } DPRINT("Total Offset %08x%08x\n", TotalOffset.HighPart, TotalOffset.LowPart); /* Lock the segment */ MmLockSectionSegment(Segment); /* Get the entry corresponding to the offset within the section */ Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset); Attributes = PAGE_READONLY; if (Required->State && Required->Page[0]) { DPRINT("Have file and page, set page %x in section @ %x #\n", Required->Page[0], TotalOffset.LowPart); if (Required->SwapEntry) MmSetSavedSwapEntryPage(Required->Page[0], Required->SwapEntry); if (Required->State & 2) { DPRINT("Set in section @ %x\n", TotalOffset.LowPart); Status = MmSetPageEntrySectionSegment(Segment, &TotalOffset, Entry = MAKE_PFN_SSE(Required->Page[0])); if (!NT_SUCCESS(Status)) { MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); } MmUnlockSectionSegment(Segment); MiSetPageEvent(Process, Address); DPRINT("Status %x\n", Status); return STATUS_MM_RESTART_OPERATION; } else { DPRINT("Set %x in address space @ %p\n", Required->Page[0], Address); Status = MmCreateVirtualMapping(Process, Address, Attributes, Required->Page, 1); if (NT_SUCCESS(Status)) { MmInsertRmap(Required->Page[0], Process, Address); } else { /* Drop the reference for our address space ... */ MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); } MmUnlockSectionSegment(Segment); DPRINTC("XXX Set Event %x\n", Status); MiSetPageEvent(Process, Address); DPRINT("Status %x\n", Status); return Status; } } else if (MM_IS_WAIT_PTE(Entry)) { // Whenever MM_WAIT_ENTRY is required as a swap entry, we need to // ask the fault handler to wait until we should continue. Rathern // than recopy this boilerplate code everywhere, we just ask them // to wait. MmUnlockSectionSegment(Segment); return STATUS_SUCCESS + 1; } else if (Entry) { PFN_NUMBER Page = PFN_FROM_SSE(Entry); DPRINT("Take reference to page %x #\n", Page); if (MiGetPfnEntry(Page) == NULL) { DPRINT1("Found no PFN entry for page 0x%x in page entry 0x%x (segment: 0x%p, offset: %08x%08x)\n", Page, Entry, Segment, TotalOffset.HighPart, TotalOffset.LowPart); KeBugCheck(CACHE_MANAGER); } OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmReferencePage(Page); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); Status = MmCreateVirtualMapping(Process, Address, Attributes, &Page, 1); if (NT_SUCCESS(Status)) { MmInsertRmap(Page, Process, Address); } DPRINT("XXX Set Event %x\n", Status); MiSetPageEvent(Process, Address); MmUnlockSectionSegment(Segment); DPRINT("Status %x\n", Status); return Status; } else { DPRINT("Get page into section\n"); /* * If the entry is zero (and it can't change because we have * locked the segment) then we need to load the page. */ //DPRINT1("Read from file %08x %wZ\n", FileOffset.LowPart, &Section->FileObject->FileName); Required->State = 2; Required->Context = Segment->FileObject; Required->Consumer = Consumer; Required->FileOffset = FileOffset; Required->Amount = PAGE_SIZE; Required->DoAcquisition = MiReadFilePage; MmSetPageEntrySectionSegment(Segment, &TotalOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); MmUnlockSectionSegment(Segment); return STATUS_MORE_PROCESSING_REQUIRED; } ASSERT(FALSE); return STATUS_ACCESS_VIOLATION; }
NTSTATUS NTAPI MmFinalizeSectionPageOut(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, BOOLEAN Dirty) { NTSTATUS Status = STATUS_SUCCESS; BOOLEAN WriteZero = FALSE, WritePage = FALSE; SWAPENTRY Swap = MmGetSavedSwapEntryPage(Page); /* Bail early if the reference count isn't where we need it */ if (MmGetReferenceCountPage(Page) != 1) { DPRINT1("Cannot page out locked page %x with ref count %lu\n", Page, MmGetReferenceCountPage(Page)); return STATUS_UNSUCCESSFUL; } MmLockSectionSegment(Segment); (void)InterlockedIncrementUL(&Segment->ReferenceCount); if (Dirty) { DPRINT("Finalize (dirty) Segment %p Page %x\n", Segment, Page); DPRINT("Segment->FileObject %p\n", Segment->FileObject); DPRINT("Segment->Flags %x\n", Segment->Flags); WriteZero = TRUE; WritePage = TRUE; } else { WriteZero = TRUE; } DPRINT("Status %x\n", Status); MmUnlockSectionSegment(Segment); if (WritePage) { DPRINT("MiWriteBackPage(Segment %p FileObject %p Offset %x)\n", Segment, Segment->FileObject, FileOffset->LowPart); Status = MiWriteBackPage(Segment->FileObject, FileOffset, PAGE_SIZE, Page); } MmLockSectionSegment(Segment); if (WriteZero && NT_SUCCESS(Status)) { DPRINT("Setting page entry in segment %p:%x to swap %x\n", Segment, FileOffset->LowPart, Swap); MmSetPageEntrySectionSegment(Segment, FileOffset, Swap ? MAKE_SWAP_SSE(Swap) : 0); } else { DPRINT("Setting page entry in segment %p:%x to page %x\n", Segment, FileOffset->LowPart, Page); MmSetPageEntrySectionSegment(Segment, FileOffset, Page ? (Dirty ? DIRTY_SSE(MAKE_PFN_SSE(Page)) : MAKE_PFN_SSE(Page)) : 0); } if (NT_SUCCESS(Status)) { DPRINT("Removing page %x for real\n", Page); MmSetSavedSwapEntryPage(Page, 0); MmReleasePageMemoryConsumer(MC_CACHE, Page); } MmUnlockSectionSegment(Segment); if (InterlockedDecrementUL(&Segment->ReferenceCount) == 0) { MmFinalizeSegment(Segment); } /* Note: Writing may evict the segment... Nothing is guaranteed from here down */ MiSetPageEvent(Segment, FileOffset->LowPart); DPRINT("Status %x\n", Status); return Status; }
NTSTATUS NTAPI MiReadFilePage(PMMSUPPORT AddressSpace, PMEMORY_AREA MemoryArea, PMM_REQUIRED_RESOURCES RequiredResources) { PFILE_OBJECT FileObject = RequiredResources->Context; PPFN_NUMBER Page = &RequiredResources->Page[RequiredResources->Offset]; PLARGE_INTEGER FileOffset = &RequiredResources->FileOffset; NTSTATUS Status; PVOID PageBuf = NULL; KEVENT Event; IO_STATUS_BLOCK IOSB; UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)]; PMDL Mdl = (PMDL)MdlBase; KIRQL OldIrql; DPRINTC("Pulling page %I64x from %wZ to %Ix\n", FileOffset->QuadPart, &FileObject->FileName, *Page); Status = MmRequestPageMemoryConsumer(RequiredResources->Consumer, TRUE, Page); if (!NT_SUCCESS(Status)) { DPRINT1("Status: %x\n", Status); return Status; } MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, Page); Mdl->MdlFlags |= MDL_PAGES_LOCKED; KeInitializeEvent(&Event, NotificationEvent, FALSE); Status = IoPageRead(FileObject, Mdl, FileOffset, &Event, &IOSB); if (Status == STATUS_PENDING) { KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); Status = IOSB.Status; } if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } PageBuf = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql); if (!PageBuf) { MmReleasePageMemoryConsumer(RequiredResources->Consumer, *Page); return STATUS_NO_MEMORY; } RtlZeroMemory((PCHAR)PageBuf+RequiredResources->Amount, PAGE_SIZE-RequiredResources->Amount); MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageBuf, OldIrql); DPRINT("Read Status %x (Page %x)\n", Status, *Page); if (!NT_SUCCESS(Status)) { MmReleasePageMemoryConsumer(RequiredResources->Consumer, *Page); DPRINT("Status: %x\n", Status); return Status; } return STATUS_SUCCESS; }