NTSTATUS NTAPI MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) { PMM_ALLOCATION_REQUEST Request; PLIST_ENTRY Entry; KIRQL OldIrql; if (Page == 0) { DPRINT1("Tried to release page zero.\n"); KeBugCheck(MEMORY_MANAGEMENT); } KeAcquireSpinLock(&AllocationListLock, &OldIrql); if (MmGetReferenceCountPage(Page) == 1) { (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages) { KeReleaseSpinLock(&AllocationListLock, OldIrql); if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmDereferencePage(Page); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { Entry = RemoveHeadList(&AllocationListHead); Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry); KeReleaseSpinLock(&AllocationListLock, OldIrql); if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); MiZeroPhysicalPage(Page); Request->Page = Page; KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE); } } else { KeReleaseSpinLock(&AllocationListLock, OldIrql); if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmDereferencePage(Page); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } return(STATUS_SUCCESS); }
NTSTATUS NTAPI MmpPageOutPhysicalAddress(PFN_NUMBER Page) { BOOLEAN ProcRef = FALSE, PageDirty; PFN_NUMBER SectionPage = 0; PMM_RMAP_ENTRY entry; PMM_SECTION_SEGMENT Segment = NULL; LARGE_INTEGER FileOffset; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace = NULL; BOOLEAN Dirty = FALSE; PVOID Address = NULL; PEPROCESS Process = NULL; NTSTATUS Status = STATUS_SUCCESS; MM_REQUIRED_RESOURCES Resources = { 0 }; DPRINTC("Page out %x (ref ct %x)\n", Page, MmGetReferenceCountPage(Page)); ExAcquireFastMutex(&MiGlobalPageOperation); if ((Segment = MmGetSectionAssociation(Page, &FileOffset))) { DPRINTC("Withdrawing page (%x) %p:%x\n", Page, Segment, FileOffset.LowPart); SectionPage = MmWithdrawSectionPage(Segment, &FileOffset, &Dirty); DPRINTC("SectionPage %x\n", SectionPage); if (SectionPage == MM_WAIT_ENTRY || SectionPage == 0) { DPRINT1("In progress page out %x\n", SectionPage); ExReleaseFastMutex(&MiGlobalPageOperation); return STATUS_UNSUCCESSFUL; } else { ASSERT(SectionPage == Page); } Resources.State = Dirty ? 1 : 0; } else { DPRINT("No segment association for %x\n", Page); } Dirty = MmIsDirtyPageRmap(Page); DPRINTC("Trying to unmap all instances of %x\n", Page); ExAcquireFastMutex(&RmapListLock); entry = MmGetRmapListHeadPage(Page); // Entry and Segment might be null here in the case that the page // is new and is in the process of being swapped in if (!entry && !Segment) { Status = STATUS_UNSUCCESSFUL; DPRINT1("Page %x is in transit\n", Page); ExReleaseFastMutex(&RmapListLock); goto bail; } while (entry != NULL && NT_SUCCESS(Status)) { Process = entry->Process; Address = entry->Address; DPRINTC("Process %p Address %p Page %x\n", Process, Address, Page); if (RMAP_IS_SEGMENT(Address)) { entry = entry->Next; continue; } if (Process && Address < MmSystemRangeStart) { /* Make sure we don't try to page out part of an exiting process */ if (PspIsProcessExiting(Process)) { DPRINT("bail\n"); ExReleaseFastMutex(&RmapListLock); goto bail; } ObReferenceObject(Process); ProcRef = TRUE; AddressSpace = &Process->Vm; } else { AddressSpace = MmGetKernelAddressSpace(); } ExReleaseFastMutex(&RmapListLock); RtlZeroMemory(&Resources, sizeof(Resources)); if ((((ULONG_PTR)Address) & 0xFFF) != 0) { KeBugCheck(MEMORY_MANAGEMENT); } do { MmLockAddressSpace(AddressSpace); MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { Status = STATUS_UNSUCCESSFUL; MmUnlockAddressSpace(AddressSpace); DPRINTC("bail\n"); goto bail; } DPRINTC("Type %x (%p -> %p)\n", MemoryArea->Type, MemoryArea->StartingAddress, MemoryArea->EndingAddress); Resources.DoAcquisition = NULL; Resources.Page[0] = Page; ASSERT(KeGetCurrentIrql() <= APC_LEVEL); DPRINT("%p:%p, page %x %x\n", Process, Address, Page, Resources.Page[0]); PageDirty = FALSE; Status = MmPageOutCacheSection(AddressSpace, MemoryArea, Address, &PageDirty, &Resources); Dirty |= PageDirty; DPRINT("%x\n", Status); ASSERT(KeGetCurrentIrql() <= APC_LEVEL); MmUnlockAddressSpace(AddressSpace); if (Status == STATUS_SUCCESS + 1) { // Wait page ... the other guy has it, so we'll just fail for now DPRINT1("Wait entry ... can't continue\n"); Status = STATUS_UNSUCCESSFUL; goto bail; } else if (Status == STATUS_MORE_PROCESSING_REQUIRED) { DPRINTC("DoAcquisition %p\n", Resources.DoAcquisition); Status = Resources.DoAcquisition(AddressSpace, MemoryArea, &Resources); DPRINTC("Status %x\n", Status); if (!NT_SUCCESS(Status)) { DPRINT1("bail\n"); goto bail; } else { Status = STATUS_MM_RESTART_OPERATION; } } } while (Status == STATUS_MM_RESTART_OPERATION); if (ProcRef) { ObDereferenceObject(Process); ProcRef = FALSE; } ExAcquireFastMutex(&RmapListLock); ASSERT(!MM_IS_WAIT_PTE(MmGetPfnForProcess(Process, Address))); entry = MmGetRmapListHeadPage(Page); DPRINTC("Entry %p\n", entry); } ExReleaseFastMutex(&RmapListLock); bail: DPRINTC("BAIL %x\n", Status); if (Segment) { ULONG RefCount; DPRINTC("About to finalize section page %x (%p:%x) Status %x %s\n", Page, Segment, FileOffset.LowPart, Status, Dirty ? "dirty" : "clean"); if (!NT_SUCCESS(Status) || !NT_SUCCESS(Status = MmFinalizeSectionPageOut(Segment, &FileOffset, Page, Dirty))) { DPRINTC("Failed to page out %x, replacing %x at %x in segment %x\n", SectionPage, FileOffset.LowPart, Segment); MmLockSectionSegment(Segment); MmSetPageEntrySectionSegment(Segment, &FileOffset, Dirty ? MAKE_PFN_SSE(Page) : DIRTY_SSE(MAKE_PFN_SSE(Page))); MmUnlockSectionSegment(Segment); } /* Alas, we had the last reference */ if ((RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0) MmFinalizeSegment(Segment); } if (ProcRef) { DPRINTC("Dereferencing process...\n"); ObDereferenceObject(Process); } ExReleaseFastMutex(&MiGlobalPageOperation); DPRINTC("%s %x %x\n", NT_SUCCESS(Status) ? "Evicted" : "Spared", Page, Status); return NT_SUCCESS(Status) ? STATUS_SUCCESS : STATUS_UNSUCCESSFUL; }
NTSTATUS NTAPI MmFinalizeSectionPageOut(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, BOOLEAN Dirty) { NTSTATUS Status = STATUS_SUCCESS; BOOLEAN WriteZero = FALSE, WritePage = FALSE; SWAPENTRY Swap = MmGetSavedSwapEntryPage(Page); /* Bail early if the reference count isn't where we need it */ if (MmGetReferenceCountPage(Page) != 1) { DPRINT1("Cannot page out locked page %x with ref count %lu\n", Page, MmGetReferenceCountPage(Page)); return STATUS_UNSUCCESSFUL; } MmLockSectionSegment(Segment); (void)InterlockedIncrementUL(&Segment->ReferenceCount); if (Dirty) { DPRINT("Finalize (dirty) Segment %p Page %x\n", Segment, Page); DPRINT("Segment->FileObject %p\n", Segment->FileObject); DPRINT("Segment->Flags %x\n", Segment->Flags); WriteZero = TRUE; WritePage = TRUE; } else { WriteZero = TRUE; } DPRINT("Status %x\n", Status); MmUnlockSectionSegment(Segment); if (WritePage) { DPRINT("MiWriteBackPage(Segment %p FileObject %p Offset %x)\n", Segment, Segment->FileObject, FileOffset->LowPart); Status = MiWriteBackPage(Segment->FileObject, FileOffset, PAGE_SIZE, Page); } MmLockSectionSegment(Segment); if (WriteZero && NT_SUCCESS(Status)) { DPRINT("Setting page entry in segment %p:%x to swap %x\n", Segment, FileOffset->LowPart, Swap); MmSetPageEntrySectionSegment(Segment, FileOffset, Swap ? MAKE_SWAP_SSE(Swap) : 0); } else { DPRINT("Setting page entry in segment %p:%x to page %x\n", Segment, FileOffset->LowPart, Page); MmSetPageEntrySectionSegment(Segment, FileOffset, Page ? (Dirty ? DIRTY_SSE(MAKE_PFN_SSE(Page)) : MAKE_PFN_SSE(Page)) : 0); } if (NT_SUCCESS(Status)) { DPRINT("Removing page %x for real\n", Page); MmSetSavedSwapEntryPage(Page, 0); MmReleasePageMemoryConsumer(MC_CACHE, Page); } MmUnlockSectionSegment(Segment); if (InterlockedDecrementUL(&Segment->ReferenceCount) == 0) { MmFinalizeSegment(Segment); } /* Note: Writing may evict the segment... Nothing is guaranteed from here down */ MiSetPageEvent(Segment, FileOffset->LowPart); DPRINT("Status %x\n", Status); return Status; }