NTSTATUS
NTAPI
MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
{
PMM_ALLOCATION_REQUEST Request;
PLIST_ENTRY Entry;
KIRQL OldIrql;
if (Page == 0)
{
DPRINT1("Tried to release page zero.\n");
KeBugCheck(MEMORY_MANAGEMENT);
}
KeAcquireSpinLock(&AllocationListLock, &OldIrql);
if (MmGetReferenceCountPage(Page) == 1)
{
(void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
{
KeReleaseSpinLock(&AllocationListLock, OldIrql);
if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
MmDereferencePage(Page);
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
else
{
Entry = RemoveHeadList(&AllocationListHead);
Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
KeReleaseSpinLock(&AllocationListLock, OldIrql);
if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
MiZeroPhysicalPage(Page);
Request->Page = Page;
KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
}
}
else
{
KeReleaseSpinLock(&AllocationListLock, OldIrql);
if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
MmDereferencePage(Page);
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
return(STATUS_SUCCESS);
}
VOID
NTAPI
PspDestroyQuotaBlock(PEPROCESS Process)
{
PEPROCESS_QUOTA_BLOCK QuotaBlock = Process->QuotaBlock;
KIRQL OldIrql;
if (QuotaBlock != &PspDefaultQuotaBlock &&
InterlockedDecrementUL(&QuotaBlock->ReferenceCount) == 0)
{
KeAcquireSpinLock(&PspQuotaLock, &OldIrql);
RemoveEntryList(&QuotaBlock->QuotaList);
KeReleaseSpinLock(&PspQuotaLock, OldIrql);
ExFreePool(QuotaBlock);
}
}
NTSTATUS
NTAPI
MmpPageOutPhysicalAddress(PFN_NUMBER Page)
{
BOOLEAN ProcRef = FALSE, PageDirty;
PFN_NUMBER SectionPage = 0;
PMM_RMAP_ENTRY entry;
PMM_SECTION_SEGMENT Segment = NULL;
LARGE_INTEGER FileOffset;
PMEMORY_AREA MemoryArea;
PMMSUPPORT AddressSpace = NULL;
BOOLEAN Dirty = FALSE;
PVOID Address = NULL;
PEPROCESS Process = NULL;
NTSTATUS Status = STATUS_SUCCESS;
MM_REQUIRED_RESOURCES Resources = { 0 };
DPRINTC("Page out %x (ref ct %x)\n", Page, MmGetReferenceCountPage(Page));
ExAcquireFastMutex(&MiGlobalPageOperation);
if ((Segment = MmGetSectionAssociation(Page, &FileOffset)))
{
DPRINTC("Withdrawing page (%x) %p:%x\n",
Page,
Segment,
FileOffset.LowPart);
SectionPage = MmWithdrawSectionPage(Segment, &FileOffset, &Dirty);
DPRINTC("SectionPage %x\n", SectionPage);
if (SectionPage == MM_WAIT_ENTRY || SectionPage == 0)
{
DPRINT1("In progress page out %x\n", SectionPage);
ExReleaseFastMutex(&MiGlobalPageOperation);
return STATUS_UNSUCCESSFUL;
}
else
{
ASSERT(SectionPage == Page);
}
Resources.State = Dirty ? 1 : 0;
}
else
{
DPRINT("No segment association for %x\n", Page);
}
Dirty = MmIsDirtyPageRmap(Page);
DPRINTC("Trying to unmap all instances of %x\n", Page);
ExAcquireFastMutex(&RmapListLock);
entry = MmGetRmapListHeadPage(Page);
// Entry and Segment might be null here in the case that the page
// is new and is in the process of being swapped in
if (!entry && !Segment)
{
Status = STATUS_UNSUCCESSFUL;
DPRINT1("Page %x is in transit\n", Page);
ExReleaseFastMutex(&RmapListLock);
goto bail;
}
while (entry != NULL && NT_SUCCESS(Status))
{
Process = entry->Process;
Address = entry->Address;
DPRINTC("Process %p Address %p Page %x\n", Process, Address, Page);
if (RMAP_IS_SEGMENT(Address))
{
entry = entry->Next;
continue;
}
if (Process && Address < MmSystemRangeStart)
{
/* Make sure we don't try to page out part of an exiting process */
if (PspIsProcessExiting(Process))
{
DPRINT("bail\n");
ExReleaseFastMutex(&RmapListLock);
goto bail;
}
ObReferenceObject(Process);
ProcRef = TRUE;
AddressSpace = &Process->Vm;
}
else
{
AddressSpace = MmGetKernelAddressSpace();
}
ExReleaseFastMutex(&RmapListLock);
RtlZeroMemory(&Resources, sizeof(Resources));
if ((((ULONG_PTR)Address) & 0xFFF) != 0)
{
KeBugCheck(MEMORY_MANAGEMENT);
}
do
{
MmLockAddressSpace(AddressSpace);
MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
{
Status = STATUS_UNSUCCESSFUL;
MmUnlockAddressSpace(AddressSpace);
DPRINTC("bail\n");
goto bail;
}
DPRINTC("Type %x (%p -> %p)\n",
MemoryArea->Type,
MemoryArea->StartingAddress,
MemoryArea->EndingAddress);
Resources.DoAcquisition = NULL;
Resources.Page[0] = Page;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
DPRINT("%p:%p, page %x %x\n",
Process,
Address,
Page,
Resources.Page[0]);
PageDirty = FALSE;
Status = MmPageOutCacheSection(AddressSpace,
MemoryArea,
Address,
&PageDirty,
&Resources);
Dirty |= PageDirty;
DPRINT("%x\n", Status);
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
MmUnlockAddressSpace(AddressSpace);
if (Status == STATUS_SUCCESS + 1)
{
// Wait page ... the other guy has it, so we'll just fail for now
DPRINT1("Wait entry ... can't continue\n");
Status = STATUS_UNSUCCESSFUL;
goto bail;
}
else if (Status == STATUS_MORE_PROCESSING_REQUIRED)
{
DPRINTC("DoAcquisition %p\n", Resources.DoAcquisition);
Status = Resources.DoAcquisition(AddressSpace,
MemoryArea,
&Resources);
DPRINTC("Status %x\n", Status);
if (!NT_SUCCESS(Status))
{
DPRINT1("bail\n");
goto bail;
}
else
{
Status = STATUS_MM_RESTART_OPERATION;
}
}
}
while (Status == STATUS_MM_RESTART_OPERATION);
if (ProcRef)
{
ObDereferenceObject(Process);
ProcRef = FALSE;
}
ExAcquireFastMutex(&RmapListLock);
ASSERT(!MM_IS_WAIT_PTE(MmGetPfnForProcess(Process, Address)));
entry = MmGetRmapListHeadPage(Page);
DPRINTC("Entry %p\n", entry);
}
ExReleaseFastMutex(&RmapListLock);
bail:
DPRINTC("BAIL %x\n", Status);
if (Segment)
{
ULONG RefCount;
DPRINTC("About to finalize section page %x (%p:%x) Status %x %s\n",
Page,
Segment,
FileOffset.LowPart,
Status,
Dirty ? "dirty" : "clean");
if (!NT_SUCCESS(Status) ||
!NT_SUCCESS(Status = MmFinalizeSectionPageOut(Segment,
&FileOffset,
Page,
Dirty)))
{
DPRINTC("Failed to page out %x, replacing %x at %x in segment %x\n",
SectionPage,
FileOffset.LowPart,
Segment);
MmLockSectionSegment(Segment);
MmSetPageEntrySectionSegment(Segment,
&FileOffset,
Dirty ? MAKE_PFN_SSE(Page) : DIRTY_SSE(MAKE_PFN_SSE(Page)));
MmUnlockSectionSegment(Segment);
}
/* Alas, we had the last reference */
if ((RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0)
MmFinalizeSegment(Segment);
}
if (ProcRef)
{
DPRINTC("Dereferencing process...\n");
ObDereferenceObject(Process);
}
ExReleaseFastMutex(&MiGlobalPageOperation);
DPRINTC("%s %x %x\n",
NT_SUCCESS(Status) ? "Evicted" : "Spared",
Page,
Status);
return NT_SUCCESS(Status) ? STATUS_SUCCESS : STATUS_UNSUCCESSFUL;
}
NTSTATUS
NTAPI
MmFinalizeSectionPageOut(PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER FileOffset,
PFN_NUMBER Page,
BOOLEAN Dirty)
{
NTSTATUS Status = STATUS_SUCCESS;
BOOLEAN WriteZero = FALSE, WritePage = FALSE;
SWAPENTRY Swap = MmGetSavedSwapEntryPage(Page);
/* Bail early if the reference count isn't where we need it */
if (MmGetReferenceCountPage(Page) != 1)
{
DPRINT1("Cannot page out locked page %x with ref count %lu\n",
Page,
MmGetReferenceCountPage(Page));
return STATUS_UNSUCCESSFUL;
}
MmLockSectionSegment(Segment);
(void)InterlockedIncrementUL(&Segment->ReferenceCount);
if (Dirty)
{
DPRINT("Finalize (dirty) Segment %p Page %x\n", Segment, Page);
DPRINT("Segment->FileObject %p\n", Segment->FileObject);
DPRINT("Segment->Flags %x\n", Segment->Flags);
WriteZero = TRUE;
WritePage = TRUE;
}
else
{
WriteZero = TRUE;
}
DPRINT("Status %x\n", Status);
MmUnlockSectionSegment(Segment);
if (WritePage)
{
DPRINT("MiWriteBackPage(Segment %p FileObject %p Offset %x)\n",
Segment,
Segment->FileObject,
FileOffset->LowPart);
Status = MiWriteBackPage(Segment->FileObject,
FileOffset,
PAGE_SIZE,
Page);
}
MmLockSectionSegment(Segment);
if (WriteZero && NT_SUCCESS(Status))
{
DPRINT("Setting page entry in segment %p:%x to swap %x\n",
Segment,
FileOffset->LowPart,
Swap);
MmSetPageEntrySectionSegment(Segment,
FileOffset,
Swap ? MAKE_SWAP_SSE(Swap) : 0);
}
else
{
DPRINT("Setting page entry in segment %p:%x to page %x\n",
Segment,
FileOffset->LowPart,
Page);
MmSetPageEntrySectionSegment(Segment,
FileOffset,
Page ? (Dirty ? DIRTY_SSE(MAKE_PFN_SSE(Page)) : MAKE_PFN_SSE(Page)) : 0);
}
if (NT_SUCCESS(Status))
{
DPRINT("Removing page %x for real\n", Page);
MmSetSavedSwapEntryPage(Page, 0);
MmReleasePageMemoryConsumer(MC_CACHE, Page);
}
MmUnlockSectionSegment(Segment);
if (InterlockedDecrementUL(&Segment->ReferenceCount) == 0)
{
MmFinalizeSegment(Segment);
}
/* Note: Writing may evict the segment... Nothing is guaranteed from here down */
MiSetPageEvent(Segment, FileOffset->LowPart);
DPRINT("Status %x\n", Status);
return Status;
}
NTSTATUS
NTAPI
MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
PPFN_NUMBER AllocatedPage)
{
ULONG OldUsed;
PFN_NUMBER Page;
KIRQL OldIrql;
/*
* Make sure we don't exceed our individual target.
*/
OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
!MiIsBalancerThread())
{
if (!CanWait)
{
(void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
return(STATUS_NO_MEMORY);
}
MiTrimMemoryConsumer(Consumer);
}
/*
* Allocate always memory for the non paged pool and for the pager thread.
*/
if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
{
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
Page = MmAllocPage(Consumer);
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
if (Page == 0)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
*AllocatedPage = Page;
if (MmAvailablePages <= MiMinimumAvailablePages &&
MiBalancerThreadHandle != NULL)
{
KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
}
return(STATUS_SUCCESS);
}
/*
* Make sure we don't exceed global targets.
*/
if (MmAvailablePages <= MiMinimumAvailablePages)
{
MM_ALLOCATION_REQUEST Request;
if (!CanWait)
{
(void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
return(STATUS_NO_MEMORY);
}
/* Insert an allocation request. */
Request.Page = 0;
KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
(void)InterlockedIncrementUL(&MiPagesRequired);
KeAcquireSpinLock(&AllocationListLock, &OldIrql);
if (MiBalancerThreadHandle != NULL)
{
KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
}
InsertTailList(&AllocationListHead, &Request.ListEntry);
KeReleaseSpinLock(&AllocationListLock, OldIrql);
KeWaitForSingleObject(&Request.Event,
0,
KernelMode,
FALSE,
NULL);
Page = Request.Page;
if (Page == 0)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
/* Update the Consumer and make the page active */
if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
(void)InterlockedDecrementUL(&MiPagesRequired);
return(STATUS_SUCCESS);
}
/*
* Actually allocate the page.
*/
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
Page = MmAllocPage(Consumer);
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
if (Page == 0)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
return(STATUS_SUCCESS);
}
/*++
* @name CsrpCheckRequestThreads
*
* The CsrpCheckRequestThreads routine checks if there are no more threads
* to handle CSR API Requests, and creates a new thread if possible, to
* avoid starvation.
*
* @param None.
*
* @return STATUS_SUCCESS in case of success, STATUS_UNSUCCESSFUL
* if a new thread couldn't be created.
*
* @remarks None.
*
*--*/
NTSTATUS
NTAPI
CsrpCheckRequestThreads(VOID)
{
HANDLE hThread;
CLIENT_ID ClientId;
NTSTATUS Status;
/* Decrease the count, and see if we're out */
if (InterlockedDecrementUL(&CsrpStaticThreadCount) == 0)
{
/* Check if we've still got space for a Dynamic Thread */
if (CsrpDynamicThreadTotal < CsrMaxApiRequestThreads)
{
/* Create a new dynamic thread */
Status = RtlCreateUserThread(NtCurrentProcess(),
NULL,
TRUE,
0,
0,
0,
(PVOID)CsrApiRequestThread,
NULL,
&hThread,
&ClientId);
/* Check success */
if (NT_SUCCESS(Status))
{
/* Increase the thread counts */
InterlockedIncrementUL(&CsrpStaticThreadCount);
InterlockedIncrementUL(&CsrpDynamicThreadTotal);
/* Add a new server thread */
if (CsrAddStaticServerThread(hThread,
&ClientId,
CsrThreadIsServerThread))
{
/* Activate it */
NtResumeThread(hThread, NULL);
}
else
{
/* Failed to create a new static thread */
InterlockedDecrementUL(&CsrpStaticThreadCount);
InterlockedDecrementUL(&CsrpDynamicThreadTotal);
/* Terminate it */
DPRINT1("Failing\n");
NtTerminateThread(hThread, 0);
NtClose(hThread);
/* Return */
return STATUS_UNSUCCESSFUL;
}
}
}
}
/* Success */
return STATUS_SUCCESS;
}
