NTSTATUS
MmAddPhysicalMemory (
IN PPHYSICAL_ADDRESS StartAddress,
IN OUT PLARGE_INTEGER NumberOfBytes
)
/*++
Routine Description:
This routine adds the specified physical address range to the system.
This includes initializing PFN database entries and adding it to the
freelists.
Arguments:
StartAddress - Supplies the starting physical address.
NumberOfBytes - Supplies a pointer to the number of bytes being added.
If any bytes were added (ie: STATUS_SUCCESS is being
returned), the actual amount is returned here.
Return Value:
NTSTATUS.
Environment:
Kernel mode. PASSIVE level. No locks held.
--*/
{
ULONG i;
PMMPFN Pfn1;
KIRQL OldIrql;
LOGICAL Inserted;
LOGICAL Updated;
MMPTE TempPte;
PMMPTE PointerPte;
PMMPTE LastPte;
PFN_NUMBER NumberOfPages;
PFN_NUMBER start;
PFN_NUMBER count;
PFN_NUMBER StartPage;
PFN_NUMBER EndPage;
PFN_NUMBER PageFrameIndex;
PFN_NUMBER Page;
PFN_NUMBER LastPage;
PFN_COUNT PagesNeeded;
PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
PPHYSICAL_MEMORY_RUN NewRun;
LOGICAL PfnDatabaseIsPhysical;
ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);
ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);
if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {
//
// The system must be configured for dynamic memory addition. This is
// critical as only then is the database guaranteed to be non-sparse.
//
if (MmDynamicPfn == FALSE) {
return STATUS_NOT_SUPPORTED;
}
PfnDatabaseIsPhysical = TRUE;
}
else {
PfnDatabaseIsPhysical = FALSE;
}
StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);
EndPage = StartPage + NumberOfPages;
if (EndPage - 1 > MmHighestPossiblePhysicalPage) {
//
// Truncate the request into something that can be mapped by the PFN
// database.
//
EndPage = MmHighestPossiblePhysicalPage + 1;
NumberOfPages = EndPage - StartPage;
}
//
// The range cannot wrap.
//
if (StartPage >= EndPage) {
return STATUS_INVALID_PARAMETER_1;
}
ExAcquireFastMutex (&MmDynamicMemoryMutex);
i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
(sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));
NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
i,
' mM');
if (NewPhysicalMemoryBlock == NULL) {
ExReleaseFastMutex (&MmDynamicMemoryMutex);
return STATUS_INSUFFICIENT_RESOURCES;
}
//
// The range cannot overlap any ranges that are already present.
//
start = 0;
LOCK_PFN (OldIrql);
do {
count = MmPhysicalMemoryBlock->Run[start].PageCount;
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
if (count != 0) {
LastPage = Page + count;
if ((StartPage < Page) && (EndPage > Page)) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_CONFLICTING_ADDRESSES;
}
if ((StartPage >= Page) && (StartPage < LastPage)) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_CONFLICTING_ADDRESSES;
}
}
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// Fill any gaps in the (sparse) PFN database needed for these pages,
// unless the PFN database was physically allocated and completely
// committed up front.
//
PagesNeeded = 0;
if (PfnDatabaseIsPhysical == FALSE) {
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
while (PointerPte <= LastPte) {
if (PointerPte->u.Hard.Valid == 0) {
PagesNeeded += 1;
}
PointerPte += 1;
}
if (MmAvailablePages < PagesNeeded) {
UNLOCK_PFN (OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (NewPhysicalMemoryBlock);
return STATUS_INSUFFICIENT_RESOURCES;
}
TempPte = ValidKernelPte;
PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
while (PointerPte <= LastPte) {
if (PointerPte->u.Hard.Valid == 0) {
PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
MiInitializePfn (PageFrameIndex, PointerPte, 0);
TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
*PointerPte = TempPte;
}
PointerPte += 1;
}
MmResidentAvailablePages -= PagesNeeded;
}
//
// If the new range is adjacent to an existing range, just merge it into
// the old block. Otherwise use the new block as a new entry will have to
// be used.
//
NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;
NewRun = &NewPhysicalMemoryBlock->Run[0];
start = 0;
Inserted = FALSE;
Updated = FALSE;
do {
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
count = MmPhysicalMemoryBlock->Run[start].PageCount;
if (Inserted == FALSE) {
//
// Note overlaps into adjacent ranges were already checked above.
//
if (StartPage == Page + count) {
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
//
// Coalesce below and above to avoid leaving zero length gaps
// as these gaps would prevent callers from removing ranges
// the span them.
//
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
start += 1;
Page = MmPhysicalMemoryBlock->Run[start].BasePage;
count = MmPhysicalMemoryBlock->Run[start].PageCount;
if (StartPage + NumberOfPages == Page) {
MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
count;
MmPhysicalMemoryBlock->NumberOfRuns -= 1;
//
// Copy any remaining entries.
//
if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
&MmPhysicalMemoryBlock->Run[start + 1],
(MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
}
}
}
Updated = TRUE;
break;
}
if (StartPage + NumberOfPages == Page) {
MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
Updated = TRUE;
break;
}
if (StartPage + NumberOfPages <= Page) {
if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {
if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
//
// Don't insert here - the new entry really belongs
// (at least) one entry further down.
//
continue;
}
}
NewRun->BasePage = StartPage;
NewRun->PageCount = NumberOfPages;
NewRun += 1;
Inserted = TRUE;
Updated = TRUE;
}
}
*NewRun = MmPhysicalMemoryBlock->Run[start];
NewRun += 1;
start += 1;
} while (start != MmPhysicalMemoryBlock->NumberOfRuns);
//
// If the memory block has not been updated, then the new entry must
// be added at the very end.
//
if (Updated == FALSE) {
ASSERT (Inserted == FALSE);
NewRun->BasePage = StartPage;
NewRun->PageCount = NumberOfPages;
Inserted = TRUE;
}
//
// Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
// releasing the PFN lock.
//
if (Inserted == TRUE) {
OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
}
//
// Note that the page directory (page parent entries on Win64) must be
// filled in at system boot so that already-created processes do not fault
// when referencing the new PFNs.
//
//
// Walk through the memory descriptors and add pages to the
// free list in the PFN database.
//
PageFrameIndex = StartPage;
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
if (EndPage - 1 > MmHighestPhysicalPage) {
MmHighestPhysicalPage = EndPage - 1;
}
while (PageFrameIndex < EndPage) {
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ShortFlags == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
ASSERT64 (Pfn1->UsedPageTableEntries == 0);
ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
ASSERT (Pfn1->PteFrame == 0);
ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
(Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));
//
// Set the PTE address to the physical page for
// virtual address alignment checking.
//
Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);
MiInsertPageInList (MmPageLocationList[FreePageList],
PageFrameIndex);
PageFrameIndex += 1;
Pfn1 += 1;
}
MmResidentAvailablePages += NumberOfPages;
MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;
UNLOCK_PFN (OldIrql);
//
// Increase all commit limits to reflect the additional memory.
//
ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
MmTotalCommitLimit += NumberOfPages;
MmTotalCommitLimitMaximum += NumberOfPages;
MmTotalCommittedPages += PagesNeeded;
ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
ExReleaseFastMutex (&MmDynamicMemoryMutex);
ExFreePool (OldPhysicalMemoryBlock);
//
// Indicate number of bytes actually added to our caller.
//
NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;
return STATUS_SUCCESS;
}
static
PMMPTE
MiGetPteForProcess(
PEPROCESS Process,
PVOID Address,
BOOLEAN Create)
{
MMPTE TmplPte, *Pte;
/* Check if we need hypersapce mapping */
if (Address < MmSystemRangeStart &&
Process && Process != PsGetCurrentProcess())
{
UNIMPLEMENTED;
__debugbreak();
return NULL;
}
else if (Create)
{
KIRQL OldIrql;
TmplPte.u.Long = 0;
TmplPte.u.Flush.Valid = 1;
TmplPte.u.Flush.Write = 1;
/* All page table levels of user pages are user owned */
TmplPte.u.Flush.Owner = (Address < MmHighestUserAddress) ? 1 : 0;
/* Lock the PFN database */
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
/* Get the PXE */
Pte = MiAddressToPxe(Address);
if (!Pte->u.Hard.Valid)
{
TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(0);
MI_WRITE_VALID_PTE(Pte, TmplPte);
}
/* Get the PPE */
Pte = MiAddressToPpe(Address);
if (!Pte->u.Hard.Valid)
{
TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(1);
MI_WRITE_VALID_PTE(Pte, TmplPte);
}
/* Get the PDE */
Pte = MiAddressToPde(Address);
if (!Pte->u.Hard.Valid)
{
TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(2);
MI_WRITE_VALID_PTE(Pte, TmplPte);
}
/* Unlock PFN database */
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
}
else
{
/* Get the PXE */
Pte = MiAddressToPxe(Address);
if (!Pte->u.Hard.Valid)
return NULL;
/* Get the PPE */
Pte = MiAddressToPpe(Address);
if (!Pte->u.Hard.Valid)
return NULL;
/* Get the PDE */
Pte = MiAddressToPde(Address);
if (!Pte->u.Hard.Valid)
return NULL;
}
return MiAddressToPte(Address);
}
LOGICAL
MiZeroAllPageFiles (
VOID
)
/*++
Routine Description:
This routine zeroes all inactive pagefile blocks in all pagefiles.
Arguments:
None.
Return Value:
Returns TRUE on success, FALSE on failure.
Environment:
Kernel mode, the caller must lock down PAGELK.
--*/
{
PMMPFN Pfn1;
PFN_NUMBER MaxPagesToWrite;
KIRQL OldIrql;
ULONG i;
PFN_NUMBER j;
PFN_NUMBER PageFrameIndex;
PMM_ZERO_PAGEFILE_CONTEXT ZeroContext;
ULONG NumberOfPagingFiles;
KEVENT WaitEvents[MAX_PAGE_FILES];
PKEVENT WaitObjects[MAX_PAGE_FILES];
KWAIT_BLOCK WaitBlockArray[MAX_PAGE_FILES];
MaxPagesToWrite = MmModifiedWriteClusterSize;
//
// Get a zeroed page to use as the source for the writes.
//
LOCK_PFN (OldIrql);
MI_DECREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_ALLOCATE_FOR_PAGEFILE_ZEROING);
if (MmAvailablePages < MM_LOW_LIMIT) {
UNLOCK_PFN (OldIrql);
MI_INCREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_FREE_FOR_PAGEFILE_ZEROING);
return TRUE;
}
PageFrameIndex = MiRemoveZeroPage (0);
Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
ASSERT (Pfn1->u2.ShareCount == 0);
ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
Pfn1->u3.e2.ReferenceCount = (USHORT) MaxPagesToWrite;
Pfn1->PteAddress = (PMMPTE) (ULONG_PTR)(X64K | 0x1);
Pfn1->OriginalPte.u.Long = 0;
MI_SET_PFN_DELETED (Pfn1);
UNLOCK_PFN (OldIrql);
//
// Capture the number of paging files in case a new one gets added.
//
NumberOfPagingFiles = MmNumberOfPagingFiles;
for (i = NumberOfPagingFiles; i != 0; i -= 1) {
KeInitializeEvent (&WaitEvents[i - 1], NotificationEvent, FALSE);
WaitObjects[i - 1] = &WaitEvents[i - 1];
ZeroContext = ExAllocatePoolWithTag (NonPagedPool,
sizeof (MM_ZERO_PAGEFILE_CONTEXT),
'wZmM');
if (ZeroContext == NULL) {
KeSetEvent (WaitObjects[i - 1], 0, FALSE);
continue;
}
ZeroContext->PagingFile = MmPagingFile[i - 1];
ZeroContext->ZeroedPageFrame = PageFrameIndex;
ZeroContext->AllDone = WaitObjects[i - 1];
if (i != 1) {
ExInitializeWorkItem (&ZeroContext->WorkItem,
MiZeroPageFile,
(PVOID) ZeroContext);
ExQueueWorkItem (&ZeroContext->WorkItem, CriticalWorkQueue);
}
else {
//
// Zero the first pagefile ourself, then wait for
// any others to finish.
//
KeSetEvent (WaitObjects[i - 1], 0, FALSE);
MiZeroPageFile (ZeroContext);
}
}
if (NumberOfPagingFiles > 1) {
KeWaitForMultipleObjects (NumberOfPagingFiles,
&WaitObjects[0],
WaitAll,
Executive,
KernelMode,
FALSE,
NULL,
&WaitBlockArray[0]);
}
LOCK_PFN (OldIrql);
ASSERT (Pfn1->u3.e2.ReferenceCount >= MaxPagesToWrite);
if (Pfn1->u3.e2.ReferenceCount == MaxPagesToWrite) {
MI_INCREMENT_RESIDENT_AVAILABLE (1, MM_RESAVAIL_FREE_FOR_PAGEFILE_ZEROING);
}
for (j = 0; j < MaxPagesToWrite; j += 1) {
MiDecrementReferenceCountInline (Pfn1, PageFrameIndex);
}
UNLOCK_PFN (OldIrql);
return TRUE;
}
