Exemplo n.º 1
0
BOOLEAN
NTAPI
MiUnProtectFreeNonPagedPool(IN PVOID VirtualAddress,
                            IN ULONG PageCount)
{
    PMMPTE PointerPte;
    MMPTE TempPte;
    PFN_NUMBER UnprotectedPages = 0;

    /* If pool is physical, can't protect PTEs */
    if (MI_IS_PHYSICAL_ADDRESS(VirtualAddress)) return FALSE;

    /* Get, and capture the PTE */
    PointerPte = MiAddressToPte(VirtualAddress);
    TempPte = *PointerPte;

    /* Loop protected PTEs */
    while ((TempPte.u.Hard.Valid == 0) && (TempPte.u.Soft.Prototype == 1))
    {
        /* Unprotect the PTE */
        TempPte.u.Hard.Valid = 1;
        TempPte.u.Soft.Prototype = 0;
        MI_WRITE_VALID_PTE(PointerPte, TempPte);

        /* One more page */
        if (++UnprotectedPages == PageCount) break;

        /* Capture next PTE */
        TempPte = *(++PointerPte);
    }

    /* Return if any pages were unprotected */
    return UnprotectedPages ? TRUE : FALSE;
}
Exemplo n.º 2
0
VOID
NTAPI
MiProtectFreeNonPagedPool(IN PVOID VirtualAddress,
                          IN ULONG PageCount)
{
    PMMPTE PointerPte, LastPte;
    MMPTE TempPte;

    /* If pool is physical, can't protect PTEs */
    if (MI_IS_PHYSICAL_ADDRESS(VirtualAddress)) return;

    /* Get PTE pointers and loop */
    PointerPte = MiAddressToPte(VirtualAddress);
    LastPte = PointerPte + PageCount;
    do
    {
        /* Capture the PTE for safety */
        TempPte = *PointerPte;

        /* Mark it as an invalid PTE, set proto bit to recognize it as pool */
        TempPte.u.Hard.Valid = 0;
        TempPte.u.Soft.Prototype = 1;
        MI_WRITE_INVALID_PTE(PointerPte, TempPte);
    } while (++PointerPte < LastPte);

    /* Flush the TLB */
    KeFlushEntireTb(TRUE, TRUE);
}
Exemplo n.º 3
0
Arquivo: pte.c Projeto: mingpen/OpenNT
ULONG
GetAddressState(
    IN PVOID VirtualAddress
    )

{
    ULONG   Address;
    ULONG   result;
    ULONG   flags = 0;
    PMMPTE  Pte;
    PMMPTE  Pde;
    ULONG   PdeContents;
    ULONG   PteContents;

    if (MI_IS_PHYSICAL_ADDRESS (VirtualAddress)) {
        return ADDRESS_VALID;
    }
    Address = (ULONG)VirtualAddress;

    Pde = (PMMPTE)MiGetPdeAddress (Address);
    Pte = (PMMPTE)MiGetPteAddress (Address);

    if ( !ReadMemory( (DWORD)Pde,
                      &PdeContents,
                      sizeof(ULONG),
                      &result) ) {
        dprintf("%08lx: Unable to get PDE\n",Pde);
        return ADDRESS_NOT_VALID;
    }

    if (PdeContents & MM_PTE_VALID_MASK) {
        if (PdeContents & MM_PTE_LARGE_PAGE_MASK) {
            return ADDRESS_VALID;
        }
        if ( !ReadMemory( (DWORD)Pte,
                          &PteContents,
                          sizeof(ULONG),
                          &result) ) {
            dprintf("%08lx: Unable to get PTE\n",Pte);
            return ADDRESS_NOT_VALID;
        }
        if (PteContents & MM_PTE_VALID_MASK) {
            return ADDRESS_VALID;
        }
        if (PteContents & MM_PTE_TRANSITION_MASK) {
            if (!(PteContents & MM_PTE_PROTOTYPE_MASK)) {
                return ADDRESS_TRANSITION;
            }
        }
    }
    return ADDRESS_NOT_VALID;
}
Exemplo n.º 4
0
/*
 * @implemented
 */
VOID
NTAPI
MmProbeAndLockPages(IN PMDL Mdl,
                    IN KPROCESSOR_MODE AccessMode,
                    IN LOCK_OPERATION Operation)
{
    PPFN_NUMBER MdlPages;
    PVOID Base, Address, LastAddress, StartAddress;
    ULONG LockPages, TotalPages;
    NTSTATUS Status = STATUS_SUCCESS;
    PEPROCESS CurrentProcess;
    NTSTATUS ProbeStatus;
    PMMPTE PointerPte, LastPte;
    PMMPDE PointerPde;
#if (_MI_PAGING_LEVELS >= 3)
    PMMPDE PointerPpe;
#endif
#if (_MI_PAGING_LEVELS == 4)
    PMMPDE PointerPxe;
#endif
    PFN_NUMBER PageFrameIndex;
    BOOLEAN UsePfnLock;
    KIRQL OldIrql;
    PMMPFN Pfn1;
    DPRINT("Probing MDL: %p\n", Mdl);

    //
    // Sanity checks
    //
    ASSERT(Mdl->ByteCount != 0);
    ASSERT(((ULONG)Mdl->ByteOffset & ~(PAGE_SIZE - 1)) == 0);
    ASSERT(((ULONG_PTR)Mdl->StartVa & (PAGE_SIZE - 1)) == 0);
    ASSERT((Mdl->MdlFlags & (MDL_PAGES_LOCKED |
                             MDL_MAPPED_TO_SYSTEM_VA |
                             MDL_SOURCE_IS_NONPAGED_POOL |
                             MDL_PARTIAL |
                             MDL_IO_SPACE)) == 0);

    //
    // Get page and base information
    //
    MdlPages = (PPFN_NUMBER)(Mdl + 1);
    Base = Mdl->StartVa;

    //
    // Get the addresses and how many pages we span (and need to lock)
    //
    Address = (PVOID)((ULONG_PTR)Base + Mdl->ByteOffset);
    LastAddress = (PVOID)((ULONG_PTR)Address + Mdl->ByteCount);
    LockPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Address, Mdl->ByteCount);
    ASSERT(LockPages != 0);

    /* Block invalid access */
    if ((AccessMode != KernelMode) &&
        ((LastAddress > (PVOID)MM_USER_PROBE_ADDRESS) || (Address >= LastAddress)))
    {
        /* Caller should be in SEH, raise the error */
        *MdlPages = LIST_HEAD;
        ExRaiseStatus(STATUS_ACCESS_VIOLATION);
    }

    //
    // Get the process
    //
    if (Address <= MM_HIGHEST_USER_ADDRESS)
    {
        //
        // Get the process
        //
        CurrentProcess = PsGetCurrentProcess();
    }
    else
    {
        //
        // No process
        //
        CurrentProcess = NULL;
    }

    //
    // Save the number of pages we'll have to lock, and the start address
    //
    TotalPages = LockPages;
    StartAddress = Address;

    /* Large pages not supported */
    ASSERT(!MI_IS_PHYSICAL_ADDRESS(Address));

    //
    // Now probe them
    //
    ProbeStatus = STATUS_SUCCESS;
    _SEH2_TRY
    {
        //
        // Enter probe loop
        //
        do
        {
            //
            // Assume failure
            //
            *MdlPages = LIST_HEAD;

            //
            // Read
            //
            *(volatile CHAR*)Address;

            //
            // Check if this is write access (only probe for user-mode)
            //
            if ((Operation != IoReadAccess) &&
                (Address <= MM_HIGHEST_USER_ADDRESS))
            {
                //
                // Probe for write too
                //
                ProbeForWriteChar(Address);
            }

            //
            // Next address...
            //
            Address = PAGE_ALIGN((ULONG_PTR)Address + PAGE_SIZE);

            //
            // Next page...
            //
            LockPages--;
            MdlPages++;
        } while (Address < LastAddress);

        //
        // Reset back to the original page
        //
        ASSERT(LockPages == 0);
        MdlPages = (PPFN_NUMBER)(Mdl + 1);
    }
    _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
    {
        //
        // Oops :(
        //
        ProbeStatus = _SEH2_GetExceptionCode();
    }
    _SEH2_END;

    //
    // So how did that go?
    //
    if (ProbeStatus != STATUS_SUCCESS)
    {
        //
        // Fail
        //
        DPRINT1("MDL PROBE FAILED!\n");
        Mdl->Process = NULL;
        ExRaiseStatus(ProbeStatus);
    }

    //
    // Get the PTE and PDE
    //
    PointerPte = MiAddressToPte(StartAddress);
    PointerPde = MiAddressToPde(StartAddress);
#if (_MI_PAGING_LEVELS >= 3)
    PointerPpe = MiAddressToPpe(StartAddress);
#endif
#if (_MI_PAGING_LEVELS == 4)
    PointerPxe = MiAddressToPxe(StartAddress);
#endif

    //
    // Sanity check
    //
    ASSERT(MdlPages == (PPFN_NUMBER)(Mdl + 1));

    //
    // Check what kind of operation this is
    //
    if (Operation != IoReadAccess)
    {
        //
        // Set the write flag
        //
        Mdl->MdlFlags |= MDL_WRITE_OPERATION;
    }
    else
    {
        //
        // Remove the write flag
        //
        Mdl->MdlFlags &= ~(MDL_WRITE_OPERATION);
    }

    //
    // Mark the MDL as locked *now*
    //
    Mdl->MdlFlags |= MDL_PAGES_LOCKED;

    //
    // Check if this came from kernel mode
    //
    if (Base > MM_HIGHEST_USER_ADDRESS)
    {
        //
        // We should not have a process
        //
        ASSERT(CurrentProcess == NULL);
        Mdl->Process = NULL;

        //
        // In kernel mode, we don't need to check for write access
        //
        Operation = IoReadAccess;

        //
        // Use the PFN lock
        //
        UsePfnLock = TRUE;
        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
    }
    else
    {
        //
        // Sanity checks
        //
        ASSERT(TotalPages != 0);
        ASSERT(CurrentProcess == PsGetCurrentProcess());

        //
        // Track locked pages
        //
        InterlockedExchangeAddSizeT(&CurrentProcess->NumberOfLockedPages,
                                    TotalPages);

        //
        // Save the process
        //
        Mdl->Process = CurrentProcess;

        /* Lock the process working set */
        MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
        UsePfnLock = FALSE;
        OldIrql = MM_NOIRQL;
    }

    //
    // Get the last PTE
    //
    LastPte = MiAddressToPte((PVOID)((ULONG_PTR)LastAddress - 1));

    //
    // Loop the pages
    //
    do
    {
        //
        // Assume failure and check for non-mapped pages
        //
        *MdlPages = LIST_HEAD;
        while (
#if (_MI_PAGING_LEVELS == 4)
               (PointerPxe->u.Hard.Valid == 0) ||
#endif
#if (_MI_PAGING_LEVELS >= 3)
               (PointerPpe->u.Hard.Valid == 0) ||
#endif
               (PointerPde->u.Hard.Valid == 0) ||
               (PointerPte->u.Hard.Valid == 0))
        {
            //
            // What kind of lock were we using?
            //
            if (UsePfnLock)
            {
                //
                // Release PFN lock
                //
                KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
            }
            else
            {
                /* Release process working set */
                MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
            }

            //
            // Access the page
            //
            Address = MiPteToAddress(PointerPte);

            //HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked
            Status = MmAccessFault(FALSE, Address, KernelMode, (PVOID)0xBADBADA3);
            if (!NT_SUCCESS(Status))
            {
                //
                // Fail
                //
                DPRINT1("Access fault failed\n");
                goto Cleanup;
            }

            //
            // What lock should we use?
            //
            if (UsePfnLock)
            {
                //
                // Grab the PFN lock
                //
                OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
            }
            else
            {
                /* Lock the process working set */
                MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
            }
        }

        //
        // Check if this was a write or modify
        //
        if (Operation != IoReadAccess)
        {
            //
            // Check if the PTE is not writable
            //
            if (MI_IS_PAGE_WRITEABLE(PointerPte) == FALSE)
            {
                //
                // Check if it's copy on write
                //
                if (MI_IS_PAGE_COPY_ON_WRITE(PointerPte))
                {
                    //
                    // Get the base address and allow a change for user-mode
                    //
                    Address = MiPteToAddress(PointerPte);
                    if (Address <= MM_HIGHEST_USER_ADDRESS)
                    {
                        //
                        // What kind of lock were we using?
                        //
                        if (UsePfnLock)
                        {
                            //
                            // Release PFN lock
                            //
                            KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
                        }
                        else
                        {
                            /* Release process working set */
                            MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
                        }

                        //
                        // Access the page
                        //

                        //HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked
                        Status = MmAccessFault(TRUE, Address, KernelMode, (PVOID)0xBADBADA3);
                        if (!NT_SUCCESS(Status))
                        {
                            //
                            // Fail
                            //
                            DPRINT1("Access fault failed\n");
                            goto Cleanup;
                        }

                        //
                        // Re-acquire the lock
                        //
                        if (UsePfnLock)
                        {
                            //
                            // Grab the PFN lock
                            //
                            OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
                        }
                        else
                        {
                            /* Lock the process working set */
                            MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
                        }

                        //
                        // Start over
                        //
                        continue;
                    }
                }

                //
                // Fail, since we won't allow this
                //
                Status = STATUS_ACCESS_VIOLATION;
                goto CleanupWithLock;
            }
        }

        //
        // Grab the PFN
        //
        PageFrameIndex = PFN_FROM_PTE(PointerPte);
        Pfn1 = MiGetPfnEntry(PageFrameIndex);
        if (Pfn1)
        {
            /* Either this is for kernel-mode, or the working set is held */
            ASSERT((CurrentProcess == NULL) || (UsePfnLock == FALSE));

            /* No Physical VADs supported yet */
            if (CurrentProcess) ASSERT(CurrentProcess->PhysicalVadRoot == NULL);

            /* This address should already exist and be fully valid */
            MiReferenceProbedPageAndBumpLockCount(Pfn1);
        }
        else
        {
            //
            // For I/O addresses, just remember this
            //
            Mdl->MdlFlags |= MDL_IO_SPACE;
        }

        //
        // Write the page and move on
        //
        *MdlPages++ = PageFrameIndex;
        PointerPte++;

        /* Check if we're on a PDE boundary */
        if (MiIsPteOnPdeBoundary(PointerPte)) PointerPde++;
#if (_MI_PAGING_LEVELS >= 3)
        if (MiIsPteOnPpeBoundary(PointerPte)) PointerPpe++;
#endif
#if (_MI_PAGING_LEVELS == 4)
        if (MiIsPteOnPxeBoundary(PointerPte)) PointerPxe++;
#endif

    } while (PointerPte <= LastPte);

    //
    // What kind of lock were we using?
    //
    if (UsePfnLock)
    {
        //
        // Release PFN lock
        //
        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
    }
    else
    {
        /* Release process working set */
        MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
    }

    //
    // Sanity check
    //
    ASSERT((Mdl->MdlFlags & MDL_DESCRIBES_AWE) == 0);
    return;

CleanupWithLock:
    //
    // This is the failure path
    //
    ASSERT(!NT_SUCCESS(Status));

    //
    // What kind of lock were we using?
    //
    if (UsePfnLock)
    {
        //
        // Release PFN lock
        //
        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
    }
    else
    {
        /* Release process working set */
        MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread());
    }
Cleanup:
    //
    // Pages must be locked so MmUnlock can work
    //
    ASSERT(Mdl->MdlFlags & MDL_PAGES_LOCKED);
    MmUnlockPages(Mdl);

    //
    // Raise the error
    //
    ExRaiseStatus(Status);
}
Exemplo n.º 5
0
VOID
MiCheckPfn (
            )

/*++

Routine Description:

    This routine checks each physical page in the PFN database to ensure
    it is in the proper state.

Arguments:

    None.

Return Value:

    None.

Environment:

    Kernel mode, APCs disabled.

--*/

{
    PMMPFN Pfn1;
    PFN_NUMBER Link, Previous;
    ULONG i;
    PMMPTE PointerPte;
    KIRQL PreviousIrql;
    KIRQL OldIrql;
    USHORT ValidCheck[4];
    USHORT ValidPage[4];
    PMMPFN PfnX;

    ValidCheck[0] = ValidCheck[1] = ValidCheck[2] = ValidCheck[3] = 0;
    ValidPage[0] = ValidPage[1] = ValidPage[2] = ValidPage[3] = 0;

    if (CheckPfnBitMap == NULL) {
        MiCreateBitMap ( &CheckPfnBitMap, MmNumberOfPhysicalPages, NonPagedPool);
    }
    RtlClearAllBits (CheckPfnBitMap);

    //
    // Walk free list.
    //

    KeRaiseIrql (APC_LEVEL, &PreviousIrql);
    LOCK_PFN (OldIrql);

    Previous = MM_EMPTY_LIST;
    Link = MmFreePageListHead.Flink;
    for (i=0; i < MmFreePageListHead.Total; i++) {
        if (Link == MM_EMPTY_LIST) {
            DbgPrint("free list total count wrong\n");
            UNLOCK_PFN (OldIrql);
            KeLowerIrql (PreviousIrql);
            return;
        }
        RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
        Pfn1 = MI_PFN_ELEMENT(Link);
        if (Pfn1->u3.e2.ReferenceCount != 0) {
            DbgPrint("non zero reference count on free list\n");
            MiFormatPfn(Pfn1);

        }
        if (Pfn1->u3.e1.PageLocation != FreePageList) {
            DbgPrint("page location not freelist\n");
            MiFormatPfn(Pfn1);
        }
        if (Pfn1->u2.Blink != Previous) {
            DbgPrint("bad blink on free list\n");
            MiFormatPfn(Pfn1);
        }
        Previous = Link;
        Link = Pfn1->u1.Flink;

    }
    if (Link != MM_EMPTY_LIST) {
            DbgPrint("free list total count wrong\n");
            Pfn1 = MI_PFN_ELEMENT(Link);
            MiFormatPfn(Pfn1);
    }

    //
    // Walk zeroed list.
    //

    Previous = MM_EMPTY_LIST;
    Link = MmZeroedPageListHead.Flink;
    for (i=0; i < MmZeroedPageListHead.Total; i++) {
        if (Link == MM_EMPTY_LIST) {
            DbgPrint("zero list total count wrong\n");
            UNLOCK_PFN (OldIrql);
            KeLowerIrql (PreviousIrql);
            return;
        }
        RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
        Pfn1 = MI_PFN_ELEMENT(Link);
        if (Pfn1->u3.e2.ReferenceCount != 0) {
            DbgPrint("non zero reference count on zero list\n");
            MiFormatPfn(Pfn1);

        }
        if (Pfn1->u3.e1.PageLocation != ZeroedPageList) {
            DbgPrint("page location not zerolist\n");
            MiFormatPfn(Pfn1);
        }
        if (Pfn1->u2.Blink != Previous) {
            DbgPrint("bad blink on zero list\n");
            MiFormatPfn(Pfn1);
        }
        Previous = Link;
        Link = Pfn1->u1.Flink;

    }
    if (Link != MM_EMPTY_LIST) {
            DbgPrint("zero list total count wrong\n");
            Pfn1 = MI_PFN_ELEMENT(Link);
            MiFormatPfn(Pfn1);
    }

    //
    // Walk Bad list.
    //
    Previous = MM_EMPTY_LIST;
    Link = MmBadPageListHead.Flink;
    for (i=0; i < MmBadPageListHead.Total; i++) {
        if (Link == MM_EMPTY_LIST) {
            DbgPrint("Bad list total count wrong\n");
            UNLOCK_PFN (OldIrql);
            KeLowerIrql (PreviousIrql);
            return;
        }
        RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
        Pfn1 = MI_PFN_ELEMENT(Link);
        if (Pfn1->u3.e2.ReferenceCount != 0) {
            DbgPrint("non zero reference count on Bad list\n");
            MiFormatPfn(Pfn1);

        }
        if (Pfn1->u3.e1.PageLocation != BadPageList) {
            DbgPrint("page location not Badlist\n");
            MiFormatPfn(Pfn1);
        }
        if (Pfn1->u2.Blink != Previous) {
            DbgPrint("bad blink on Bad list\n");
            MiFormatPfn(Pfn1);
        }
        Previous = Link;
        Link = Pfn1->u1.Flink;

    }
    if (Link != MM_EMPTY_LIST) {
            DbgPrint("Bad list total count wrong\n");
            Pfn1 = MI_PFN_ELEMENT(Link);
            MiFormatPfn(Pfn1);
    }

    //
    // Walk Standby list.
    //

    Previous = MM_EMPTY_LIST;
    Link = MmStandbyPageListHead.Flink;
    for (i=0; i < MmStandbyPageListHead.Total; i++) {
        if (Link == MM_EMPTY_LIST) {
            DbgPrint("Standby list total count wrong\n");
            UNLOCK_PFN (OldIrql);
            KeLowerIrql (PreviousIrql);
            return;
        }
        RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
        Pfn1 = MI_PFN_ELEMENT(Link);
        if (Pfn1->u3.e2.ReferenceCount != 0) {
            DbgPrint("non zero reference count on Standby list\n");
            MiFormatPfn(Pfn1);

        }
        if (Pfn1->u3.e1.PageLocation != StandbyPageList) {
            DbgPrint("page location not Standbylist\n");
            MiFormatPfn(Pfn1);
        }
        if (Pfn1->u2.Blink != Previous) {
            DbgPrint("bad blink on Standby list\n");
            MiFormatPfn(Pfn1);
        }

        //
        // Check to see if referenced PTE is okay.
        //
        if (MI_IS_PFN_DELETED (Pfn1)) {
            DbgPrint("Invalid pteaddress in standby list\n");
            MiFormatPfn(Pfn1);

        } else {

            OldIrql = 99;
            if ((Pfn1->u3.e1.PrototypePte == 1) &&
                            (MmIsAddressValid (Pfn1->PteAddress))) {
                PointerPte = Pfn1->PteAddress;
            } else {
                PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame,
                                                   &OldIrql);
                PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
                                    MiGetByteOffset(Pfn1->PteAddress));
            }
            if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
                DbgPrint("Invalid PFN - PTE address is wrong in standby list\n");
                MiFormatPfn(Pfn1);
                MiFormatPte(PointerPte);
            }
            if (PointerPte->u.Soft.Transition == 0) {
                DbgPrint("Pte not in transition for page on standby list\n");
                MiFormatPfn(Pfn1);
                MiFormatPte(PointerPte);
            }
            if (OldIrql != 99) {
                MiUnmapPageInHyperSpace (OldIrql);
                OldIrql = 99;
            }

        }

        Previous = Link;
        Link = Pfn1->u1.Flink;

    }
    if (Link != MM_EMPTY_LIST) {
            DbgPrint("Standby list total count wrong\n");
            Pfn1 = MI_PFN_ELEMENT(Link);
            MiFormatPfn(Pfn1);
    }

    //
    // Walk Modified list.
    //

    Previous = MM_EMPTY_LIST;
    Link = MmModifiedPageListHead.Flink;
    for (i=0; i < MmModifiedPageListHead.Total; i++) {
        if (Link == MM_EMPTY_LIST) {
            DbgPrint("Modified list total count wrong\n");
            UNLOCK_PFN (OldIrql);
            KeLowerIrql (PreviousIrql);
            return;
        }
        RtlSetBits (CheckPfnBitMap, (ULONG)Link, 1L);
        Pfn1 = MI_PFN_ELEMENT(Link);
        if (Pfn1->u3.e2.ReferenceCount != 0) {
            DbgPrint("non zero reference count on Modified list\n");
            MiFormatPfn(Pfn1);

        }
        if (Pfn1->u3.e1.PageLocation != ModifiedPageList) {
            DbgPrint("page location not Modifiedlist\n");
            MiFormatPfn(Pfn1);
        }
        if (Pfn1->u2.Blink != Previous) {
            DbgPrint("bad blink on Modified list\n");
            MiFormatPfn(Pfn1);
        }
        //
        // Check to see if referenced PTE is okay.
        //
        if (MI_IS_PFN_DELETED (Pfn1)) {
            DbgPrint("Invalid pteaddress in modified list\n");
            MiFormatPfn(Pfn1);

        } else {

            if ((Pfn1->u3.e1.PrototypePte == 1) &&
                            (MmIsAddressValid (Pfn1->PteAddress))) {
                PointerPte = Pfn1->PteAddress;
            } else {
                PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
                PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
                                    MiGetByteOffset(Pfn1->PteAddress));
            }

            if (MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (PointerPte) != Link) {
                DbgPrint("Invalid PFN - PTE address is wrong in modified list\n");
                MiFormatPfn(Pfn1);
                MiFormatPte(PointerPte);
            }
            if (PointerPte->u.Soft.Transition == 0) {
                DbgPrint("Pte not in transition for page on modified list\n");
                MiFormatPfn(Pfn1);
                MiFormatPte(PointerPte);
            }

            if (OldIrql != 99) {
                MiUnmapPageInHyperSpace (OldIrql);
                OldIrql = 99;
            }
        }

        Previous = Link;
        Link = Pfn1->u1.Flink;

    }
    if (Link != MM_EMPTY_LIST) {
            DbgPrint("Modified list total count wrong\n");
            Pfn1 = MI_PFN_ELEMENT(Link);
            MiFormatPfn(Pfn1);
    }
    //
    // All non active pages have been scanned.  Locate the
    // active pages and make sure they are consistent.
    //

    //
    // set bit zero as page zero is reserved for now
    //

    RtlSetBits (CheckPfnBitMap, 0L, 1L);

    Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);
    while (Link != 0xFFFFFFFF) {
        Pfn1 = MI_PFN_ELEMENT (Link);

        //
        // Make sure the PTE address is okay
        //

        if ((Pfn1->PteAddress >= (PMMPTE)HYPER_SPACE)
                && (Pfn1->u3.e1.PrototypePte == 0)) {
            DbgPrint("pfn with illegal pte address\n");
            MiFormatPfn(Pfn1);
            break;
        }

        if (Pfn1->PteAddress < (PMMPTE)PTE_BASE) {
            DbgPrint("pfn with illegal pte address\n");
            MiFormatPfn(Pfn1);
            break;
        }

#if defined(_IA64_)

        //
        // ignore PTEs mapped to IA64 kernel BAT.
        //

        if (MI_IS_PHYSICAL_ADDRESS(MiGetVirtualAddressMappedByPte(Pfn1->PteAddress))) {

            goto NoCheck;
        }
#endif // _IA64_

#ifdef _ALPHA_

        //
        // ignore ptes mapped to ALPHA's 32-bit superpage.
        //

        if ((Pfn1->PteAddress > (PMMPTE)(ULONG_PTR)0xc0100000) &&
            (Pfn1->PteAddress < (PMMPTE)(ULONG_PTR)0xc0180000)) {

            goto NoCheck;
        }
#endif //ALPHA

        //
        // Check to make sure the referenced PTE is for this page.
        //

        if ((Pfn1->u3.e1.PrototypePte == 1) &&
                            (MmIsAddressValid (Pfn1->PteAddress))) {
            PointerPte = Pfn1->PteAddress;
        } else {
            PointerPte = MiMapPageInHyperSpace(Pfn1->PteFrame, &OldIrql);
            PointerPte = (PMMPTE)((ULONG_PTR)PointerPte +
                                    MiGetByteOffset(Pfn1->PteAddress));
        }

        if (MI_GET_PAGE_FRAME_FROM_PTE (PointerPte) != Link) {
            DbgPrint("Invalid PFN - PTE address is wrong in active list\n");
            MiFormatPfn(Pfn1);
            MiFormatPte(PointerPte);
        }
        if (PointerPte->u.Hard.Valid == 0) {
            //
            // if the page is a page table page it could be out of
            // the working set yet a transition page is keeping it
            // around in memory (ups the share count).
            //

            if ((Pfn1->PteAddress < (PMMPTE)PDE_BASE) ||
                (Pfn1->PteAddress > (PMMPTE)PDE_TOP)) {

                DbgPrint("Pte not valid for page on active list\n");
                MiFormatPfn(Pfn1);
                MiFormatPte(PointerPte);
            }
        }

        if (Pfn1->u3.e2.ReferenceCount != 1) {
            DbgPrint("refcount not 1\n");
            MiFormatPfn(Pfn1);
        }


        //
        // Check to make sure the PTE count for the frame is okay.
        //

        if (Pfn1->u3.e1.PrototypePte == 1) {
            PfnX = MI_PFN_ELEMENT(Pfn1->PteFrame);
            for (i = 0; i < 4; i++) {
                if (ValidPage[i] == 0) {
                    ValidPage[i] = (USHORT)Pfn1->PteFrame;
                }
                if (ValidPage[i] == (USHORT)Pfn1->PteFrame) {
                    ValidCheck[i] += 1;
                    break;
                }
            }
        }
        if (OldIrql != 99) {
            MiUnmapPageInHyperSpace (OldIrql);
            OldIrql = 99;
        }

#if defined(_ALPHA_) || defined(_IA64_)
NoCheck:
#endif
        Link = RtlFindClearBitsAndSet (CheckPfnBitMap, 1L, 0);

    }

    for (i = 0; i < 4; i++) {
        if (ValidPage[i] == 0) {
            break;
        }
        PfnX = MI_PFN_ELEMENT(ValidPage[i]);
    }

    UNLOCK_PFN (OldIrql);
    KeLowerIrql (PreviousIrql);
    return;

}
Exemplo n.º 6
0
NTSTATUS
MiCcPutPagesInTransition (
    IN PMI_READ_INFO MiReadInfo
    )

/*++

Routine Description:

    This routine allocates physical memory for the specified read-list and
    puts all the pages in transition (so collided faults from other threads
    for these same pages remain coherent).  I/O for any pages not already
    resident are issued here.  The caller must wait for their completion.

Arguments:

    MiReadInfo - Supplies a pointer to the read-list.

Return Value:

    STATUS_SUCCESS - all the pages were already resident, reference counts
                     have been applied and no I/O needs to be waited for.

    STATUS_ISSUE_PAGING_IO - the I/O has been issued and the caller must wait.

    Various other failure status values indicate the operation failed.

Environment:

    Kernel mode. PASSIVE_LEVEL.

--*/

{
    NTSTATUS status;
    PMMPTE LocalPrototypePte;
    PVOID StartingVa;
    PFN_NUMBER MdlPages;
    KIRQL OldIrql;
    MMPTE PteContents;
    PFN_NUMBER PageFrameIndex;
    PFN_NUMBER ResidentAvailableCharge;
    PPFN_NUMBER IoPage;
    PPFN_NUMBER ApiPage;
    PPFN_NUMBER Page;
    PPFN_NUMBER DestinationPage;
    ULONG PageColor;
    PMMPTE PointerPte;
    PMMPTE *ProtoPteArray;
    PMMPTE *EndProtoPteArray;
    PFN_NUMBER DummyPage;
    PMDL Mdl;
    PMDL FreeMdl;
    PMMPFN PfnProto;
    PMMPFN Pfn1;
    PMMPFN DummyPfn1;
    ULONG i;
    PFN_NUMBER DummyTrim;
    ULONG NumberOfPagesNeedingIo;
    MMPTE TempPte;
    PMMPTE PointerPde;
    PEPROCESS CurrentProcess;
    PMMINPAGE_SUPPORT InPageSupport;
    PKPRCB Prcb;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    MiReadInfo->DummyPagePfn = NULL;

    FreeMdl = NULL;
    CurrentProcess = PsGetCurrentProcess();

    PfnProto = NULL;
    PointerPde = NULL;

    InPageSupport = MiReadInfo->InPageSupport;
    
    Mdl = MI_EXTRACT_PREFETCH_MDL (InPageSupport);
    ASSERT (Mdl == MiReadInfo->IoMdl);

    IoPage = (PPFN_NUMBER)(Mdl + 1);
    ApiPage = (PPFN_NUMBER)(MiReadInfo->ApiMdl + 1);

    StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
    
    MdlPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES (StartingVa,
                                               Mdl->ByteCount);

    if (MdlPages + 1 > MAXUSHORT) {

        //
        // The PFN ReferenceCount for the dummy page could wrap, refuse the
        // request.
        //

        return STATUS_INSUFFICIENT_RESOURCES;
    }

    NumberOfPagesNeedingIo = 0;

    ProtoPteArray = (PMMPTE *)InPageSupport->BasePte;
    EndProtoPteArray = ProtoPteArray + MdlPages;

    ASSERT (*ProtoPteArray != NULL);

    LOCK_PFN (OldIrql);

    //
    // Ensure sufficient pages exist for the transfer plus the dummy page.
    //

    if (((SPFN_NUMBER)MdlPages > (SPFN_NUMBER)(MmAvailablePages - MM_HIGH_LIMIT)) ||
        (MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MdlPages)) {

        UNLOCK_PFN (OldIrql);

        return STATUS_INSUFFICIENT_RESOURCES;
    }

    //
    // Charge resident available immediately as the PFN lock may get released
    // and reacquired below before all the pages have been locked down.
    // Note the dummy page is immediately charged separately.
    //

    MI_DECREMENT_RESIDENT_AVAILABLE (MdlPages, MM_RESAVAIL_ALLOCATE_BUILDMDL);

    ResidentAvailableCharge = MdlPages;

    //
    // Allocate a dummy page to map discarded pages that aren't skipped.
    //

    DummyPage = MiRemoveAnyPage (0);
    Pfn1 = MI_PFN_ELEMENT (DummyPage);

    ASSERT (Pfn1->u2.ShareCount == 0);
    ASSERT (Pfn1->u3.e2.ReferenceCount == 0);

    MiInitializePfnForOtherProcess (DummyPage, MI_PF_DUMMY_PAGE_PTE, 0);

    //
    // Give the page a containing frame so MiIdentifyPfn won't crash.
    //

    Pfn1->u4.PteFrame = PsInitialSystemProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT;

    //
    // Always bias the reference count by 1 and charge for this locked page
    // up front so the myriad increments and decrements don't get slowed
    // down with needless checking.
    //

    Pfn1->u3.e1.PrototypePte = 0;

    MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);

    Pfn1->u3.e1.ReadInProgress = 1;

    MiReadInfo->DummyPagePfn = Pfn1;

    DummyPfn1 = Pfn1;

    DummyPfn1->u3.e2.ReferenceCount =
        (USHORT)(DummyPfn1->u3.e2.ReferenceCount + MdlPages);

    //
    // Properly initialize the inpage support block fields we overloaded.
    //

    InPageSupport->BasePte = *ProtoPteArray;

    //
    // Build the proper InPageSupport and MDL to describe this run.
    //

    for (; ProtoPteArray < EndProtoPteArray; ProtoPteArray += 1, IoPage += 1, ApiPage += 1) {
    
        //
        // Fill the MDL entry for this RLE.
        //
    
        PointerPte = *ProtoPteArray;

        ASSERT (PointerPte != NULL);

        //
        // The PointerPte better be inside a prototype PTE allocation
        // so that subsequent page trims update the correct PTEs.
        //

        ASSERT (((PointerPte >= (PMMPTE)MmPagedPoolStart) &&
                (PointerPte <= (PMMPTE)MmPagedPoolEnd)) ||
                ((PointerPte >= (PMMPTE)MmSpecialPoolStart) && (PointerPte <= (PMMPTE)MmSpecialPoolEnd)));

        //
        // Check the state of this prototype PTE now that the PFN lock is held.
        // If the page is not resident, the PTE must be put in transition with
        // read in progress before the PFN lock is released.
        //

        //
        // Lock page containing prototype PTEs in memory by
        // incrementing the reference count for the page.
        // Unlock any page locked earlier containing prototype PTEs if
        // the containing page is not the same for both.
        //

        if (PfnProto != NULL) {

            if (PointerPde != MiGetPteAddress (PointerPte)) {

                ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
                MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
                PfnProto = NULL;
            }
        }

        if (PfnProto == NULL) {

            ASSERT (!MI_IS_PHYSICAL_ADDRESS (PointerPte));
   
            PointerPde = MiGetPteAddress (PointerPte);
 
            if (PointerPde->u.Hard.Valid == 0) {
                MiMakeSystemAddressValidPfn (PointerPte, OldIrql);
            }

            PfnProto = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
            MI_ADD_LOCKED_PAGE_CHARGE (PfnProto);
            ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
        }

recheck:
        PteContents = *PointerPte;

        // LWFIX: are zero or dzero ptes possible here ?
        ASSERT (PteContents.u.Long != 0);

        if (PteContents.u.Hard.Valid == 1) {
            PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&PteContents);
            Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
            ASSERT (Pfn1->u3.e1.PrototypePte == 1);
            MI_ADD_LOCKED_PAGE_CHARGE (Pfn1);
            *ApiPage = PageFrameIndex;
            *IoPage = DummyPage;
            continue;
        }

        if ((PteContents.u.Soft.Prototype == 0) &&
            (PteContents.u.Soft.Transition == 1)) {

            //
            // The page is in transition.  If there is an inpage still in
            // progress, wait for it to complete.  Reference the PFN and
            // then march on.
            //

            PageFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (&PteContents);
            Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
            ASSERT (Pfn1->u3.e1.PrototypePte == 1);

            if (Pfn1->u4.InPageError) {

                //
                // There was an in-page read error and there are other
                // threads colliding for this page, delay to let the
                // other threads complete and then retry.
                //

                UNLOCK_PFN (OldIrql);
                KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
                LOCK_PFN (OldIrql);
                goto recheck;
            }

            if (Pfn1->u3.e1.ReadInProgress) {
                    // LWFIX - start with temp\aw.c
            }

            //
            // PTE refers to a normal transition PTE.
            //

            ASSERT ((SPFN_NUMBER)MmAvailablePages >= 0);

            if (MmAvailablePages == 0) {

                //
                // This can only happen if the system is utilizing a hardware
                // compression cache.  This ensures that only a safe amount
                // of the compressed virtual cache is directly mapped so that
                // if the hardware gets into trouble, we can bail it out.
                //

                UNLOCK_PFN (OldIrql);
                KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond);
                LOCK_PFN (OldIrql);
                goto recheck;
            }

            //
            // The PFN reference count will be 1 already here if the
            // modified writer has begun a write of this page.  Otherwise
            // it's ordinarily 0.
            //

            MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);

            *IoPage = DummyPage;
            *ApiPage = PageFrameIndex;
            continue;
        }

        // LWFIX: need to handle protos that are now pagefile (or dzero)
        // backed - prefetching it from the file here would cause us to lose
        // the contents.  Note this can happen for session-space images
        // as we back modified (ie: for relocation fixups or IAT
        // updated) portions from the pagefile.  remove the assert below too.
        ASSERT (PteContents.u.Soft.Prototype == 1);

        if ((MmAvailablePages < MM_HIGH_LIMIT) &&
            (MiEnsureAvailablePageOrWait (NULL, OldIrql))) {

            //
            // Had to wait so recheck all state.
            //

            goto recheck;
        }

        NumberOfPagesNeedingIo += 1;

        //
        // Allocate a physical page.
        //

        PageColor = MI_PAGE_COLOR_VA_PROCESS (
                        MiGetVirtualAddressMappedByPte (PointerPte),
                        &CurrentProcess->NextPageColor);

        PageFrameIndex = MiRemoveAnyPage (PageColor);

        Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

        ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
        ASSERT (Pfn1->u2.ShareCount == 0);
        ASSERT (PointerPte->u.Hard.Valid == 0);

        //
        // Initialize read-in-progress PFN.
        //
    
        MiInitializePfn (PageFrameIndex, PointerPte, 0);

        //
        // These pieces of MiInitializePfn initialization are overridden
        // here as these pages are only going into prototype
        // transition and not into any page tables.
        //

        Pfn1->u3.e1.PrototypePte = 1;
        Pfn1->u2.ShareCount -= 1;
        ASSERT (Pfn1->u2.ShareCount == 0);
        Pfn1->u3.e1.PageLocation = ZeroedPageList;
        Pfn1->u3.e2.ReferenceCount -= 1;
        ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
        MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1);

        //
        // Initialize the I/O specific fields.
        //
    
        Pfn1->u1.Event = &InPageSupport->Event;
        Pfn1->u3.e1.ReadInProgress = 1;
        ASSERT (Pfn1->u4.InPageError == 0);

        //
        // Increment the PFN reference count in the control area for
        // the subsection.
        //

        MiReadInfo->ControlArea->NumberOfPfnReferences += 1;
    
        //
        // Put the prototype PTE into the transition state.
        //

        MI_MAKE_TRANSITION_PTE (TempPte,
                                PageFrameIndex,
                                PointerPte->u.Soft.Protection,
                                PointerPte);

        MI_WRITE_INVALID_PTE (PointerPte, TempPte);

        *IoPage = PageFrameIndex;
        *ApiPage = PageFrameIndex;
    }
    
    //
    // If all the pages were resident, dereference the dummy page references
    // now and notify our caller that I/O is not necessary.
    //
    
    if (NumberOfPagesNeedingIo == 0) {
        ASSERT (DummyPfn1->u3.e2.ReferenceCount > MdlPages);
        DummyPfn1->u3.e2.ReferenceCount =
            (USHORT)(DummyPfn1->u3.e2.ReferenceCount - MdlPages);

        //
        // Unlock page containing prototype PTEs.
        //

        if (PfnProto != NULL) {
            ASSERT (PfnProto->u3.e2.ReferenceCount > 1);
            MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto);
        }

        UNLOCK_PFN (OldIrql);

        //
        // Return the upfront resident available charge as the
        // individual charges have all been made at this point.
        //

        MI_INCREMENT_RESIDENT_AVAILABLE (ResidentAvailableCharge,
                                         MM_RESAVAIL_FREE_BUILDMDL_EXCESS);

        return STATUS_SUCCESS;
    }

    //
    // Carefully trim leading dummy pages.
    //

    Page = (PPFN_NUMBER)(Mdl + 1);

    DummyTrim = 0;
    for (i = 0; i < MdlPages - 1; i += 1) {
        if (*Page == DummyPage) {
            DummyTrim += 1;
            Page += 1;
        }
        else {
            break;
        }
    }

    if (DummyTrim != 0) {

        Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
        Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
        ASSERT (Mdl->ByteCount != 0);
        InPageSupport->ReadOffset.QuadPart += (DummyTrim * PAGE_SIZE);
        DummyPfn1->u3.e2.ReferenceCount =
                (USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);

        //
        // Shuffle down the PFNs in the MDL.
        // Recalculate BasePte to adjust for the shuffle.
        //

        Pfn1 = MI_PFN_ELEMENT (*Page);

        ASSERT (Pfn1->PteAddress->u.Hard.Valid == 0);
        ASSERT ((Pfn1->PteAddress->u.Soft.Prototype == 0) &&
                 (Pfn1->PteAddress->u.Soft.Transition == 1));

        InPageSupport->BasePte = Pfn1->PteAddress;

        DestinationPage = (PPFN_NUMBER)(Mdl + 1);

        do {
            *DestinationPage = *Page;
            DestinationPage += 1;
            Page += 1;
            i += 1;
        } while (i < MdlPages);

        MdlPages -= DummyTrim;
    }

    //
    // Carefully trim trailing dummy pages.
    //

    ASSERT (MdlPages != 0);

    Page = (PPFN_NUMBER)(Mdl + 1) + MdlPages - 1;

    if (*Page == DummyPage) {

        ASSERT (MdlPages >= 2);

        //
        // Trim the last page specially as it may be a partial page.
        //

        Mdl->Size -= sizeof(PFN_NUMBER);
        if (BYTE_OFFSET(Mdl->ByteCount) != 0) {
            Mdl->ByteCount &= ~(PAGE_SIZE - 1);
        }
        else {
            Mdl->ByteCount -= PAGE_SIZE;
        }
        ASSERT (Mdl->ByteCount != 0);
        DummyPfn1->u3.e2.ReferenceCount -= 1;

        //
        // Now trim any other trailing pages.
        //

        Page -= 1;
        DummyTrim = 0;
        while (Page != ((PPFN_NUMBER)(Mdl + 1))) {
            if (*Page != DummyPage) {
                break;
            }
            DummyTrim += 1;
            Page -= 1;
        }
        if (DummyTrim != 0) {
            ASSERT (Mdl->Size > (USHORT)(DummyTrim * sizeof(PFN_NUMBER)));
            Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER)));
            Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE);
            DummyPfn1->u3.e2.ReferenceCount =
                (USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim);
        }

        ASSERT (MdlPages > DummyTrim + 1);
        MdlPages -= (DummyTrim + 1);

#if DBG
        StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset);
    
        ASSERT (MdlPages == ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa,
                                                               Mdl->ByteCount));
#endif
    }

    //
    // If the MDL is not already embedded in the inpage block, see if its
    // final size qualifies it - if so, embed it now.
    //

    if ((Mdl != &InPageSupport->Mdl) &&
        (Mdl->ByteCount <= (MM_MAXIMUM_READ_CLUSTER_SIZE + 1) * PAGE_SIZE)){

#if DBG
        RtlFillMemoryUlong (&InPageSupport->Page[0],
                            (MM_MAXIMUM_READ_CLUSTER_SIZE+1) * sizeof (PFN_NUMBER),
                            0xf1f1f1f1);
#endif

        RtlCopyMemory (&InPageSupport->Mdl, Mdl, Mdl->Size);

        FreeMdl = Mdl;

        Mdl = &InPageSupport->Mdl;

        ASSERT (((ULONG_PTR)Mdl & (sizeof(QUAD) - 1)) == 0);
        InPageSupport->u1.e1.PrefetchMdlHighBits = ((ULONG_PTR)Mdl >> 3);
    }
Exemplo n.º 7
0
NTSTATUS
MmRemovePhysicalMemory (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes
    )

/*++

Routine Description:

    This routine attempts to remove the specified physical address range
    from the system.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being removed.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    ULONG i;
    ULONG Additional;
    PFN_NUMBER Page;
    PFN_NUMBER LastPage;
    PFN_NUMBER OriginalLastPage;
    PFN_NUMBER start;
    PFN_NUMBER PagesReleased;
    PMMPFN Pfn1;
    PMMPFN StartPfn;
    PMMPFN EndPfn;
    KIRQL OldIrql;
    PFN_NUMBER StartPage;
    PFN_NUMBER EndPage;
    PFN_COUNT NumberOfPages;
    SPFN_NUMBER MaxPages;
    PFN_NUMBER PageFrameIndex;
    PFN_NUMBER RemovedPages;
    LOGICAL Inserted;
    NTSTATUS Status;
    PMMPTE PointerPte;
    PMMPTE EndPte;
    PVOID VirtualAddress;
    PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_RUN NewRun;
    LOGICAL PfnDatabaseIsPhysical;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
    ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);

    if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {

        //
        // The system must be configured for dynamic memory addition.  This is
        // not strictly required to remove the memory, but it's better to check
        // for it now under the assumption that the administrator is probably
        // going to want to add this range of memory back in - better to give
        // the error now and refuse the removal than to refuse the addition
        // later.
        //
    
        if (MmDynamicPfn == FALSE) {
            return STATUS_NOT_SUPPORTED;
        }

        PfnDatabaseIsPhysical = TRUE;
    }
    else {
        PfnDatabaseIsPhysical = FALSE;
    }

    StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    NumberOfPages = (PFN_COUNT)(NumberOfBytes->QuadPart >> PAGE_SHIFT);

    EndPage = StartPage + NumberOfPages;

    if (EndPage - 1 > MmHighestPossiblePhysicalPage) {

        //
        // Truncate the request into something that can be mapped by the PFN
        // database.
        //

        EndPage = MmHighestPossiblePhysicalPage + 1;
        NumberOfPages = (PFN_COUNT)(EndPage - StartPage);
    }

    //
    // The range cannot wrap.
    //

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    StartPfn = MI_PFN_ELEMENT (StartPage);
    EndPfn = MI_PFN_ELEMENT (EndPage);

    ExAcquireFastMutex (&MmDynamicMemoryMutex);

#if DBG
    MiDynmemData[0] += 1;
#endif

    //
    // Decrease all commit limits to reflect the removed memory.
    //

    ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);

    ASSERT (MmTotalCommitLimit <= MmTotalCommitLimitMaximum);

    if ((NumberOfPages + 100 > MmTotalCommitLimit - MmTotalCommittedPages) ||
        (MmTotalCommittedPages > MmTotalCommitLimit)) {

#if DBG
        MiDynmemData[1] += 1;
#endif
        ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);
        ExReleaseFastMutex (&MmDynamicMemoryMutex);
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    MmTotalCommitLimit -= NumberOfPages;
    MmTotalCommitLimitMaximum -= NumberOfPages;

    ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);

    //
    // Check for outstanding promises that cannot be broken.
    //

    LOCK_PFN (OldIrql);

    MaxPages = MI_NONPAGABLE_MEMORY_AVAILABLE() - 100;

    if ((SPFN_NUMBER)NumberOfPages > MaxPages) {
#if DBG
        MiDynmemData[2] += 1;
#endif
        UNLOCK_PFN (OldIrql);
        Status = STATUS_INSUFFICIENT_RESOURCES;
        goto giveup2;
    }

    MmResidentAvailablePages -= NumberOfPages;
    MmNumberOfPhysicalPages -= NumberOfPages;

    //
    // The range must be contained in a single entry.  It is permissible for
    // it to be part of a single entry, but it must not cross multiple entries.
    //

    Additional = (ULONG)-2;

    start = 0;
    do {

        Page = MmPhysicalMemoryBlock->Run[start].BasePage;
        LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;

        if ((StartPage >= Page) && (EndPage <= LastPage)) {
            if ((StartPage == Page) && (EndPage == LastPage)) {
                Additional = (ULONG)-1;
            }
            else if ((StartPage == Page) || (EndPage == LastPage)) {
                Additional = 0;
            }
            else {
                Additional = 1;
            }
            break;
        }

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    if (Additional == (ULONG)-2) {
#if DBG
        MiDynmemData[3] += 1;
#endif
        MmResidentAvailablePages += NumberOfPages;
        MmNumberOfPhysicalPages += NumberOfPages;
        UNLOCK_PFN (OldIrql);
        Status = STATUS_CONFLICTING_ADDRESSES;
        goto giveup2;
    }

    for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
        Pfn1->u3.e1.RemovalRequested = 1;
    }

    //
    // The free and zero lists must be pruned now before releasing the PFN
    // lock otherwise if another thread allocates the page from these lists,
    // the allocation will clear the RemovalRequested flag forever.
    //

    RemovedPages = MiRemovePhysicalPages (StartPage, EndPage);

    if (RemovedPages != NumberOfPages) {

#if DBG
retry:
#endif
    
        Pfn1 = StartPfn;
    
        InterlockedIncrement (&MiDelayPageFaults);
    
        for (i = 0; i < 5; i += 1) {
    
            UNLOCK_PFN (OldIrql);
    
            //
            // Attempt to move pages to the standby list.  Note that only the
            // pages with RemovalRequested set are moved.
            //
    
            MiTrimRemovalPagesOnly = TRUE;
    
            MiEmptyAllWorkingSets ();
    
            MiTrimRemovalPagesOnly = FALSE;
    
            MiFlushAllPages ();
    
            KeDelayExecutionThread (KernelMode, FALSE, &MmHalfSecond);
    
            LOCK_PFN (OldIrql);
    
            RemovedPages += MiRemovePhysicalPages (StartPage, EndPage);
    
            if (RemovedPages == NumberOfPages) {
                break;
            }
    
            //
            // RemovedPages doesn't include pages that were freed directly to
            // the bad page list via MiDecrementReferenceCount.  So use the above
            // check purely as an optimization - and walk here when necessary.
            //
    
            for ( ; Pfn1 < EndPfn; Pfn1 += 1) {
                if (Pfn1->u3.e1.PageLocation != BadPageList) {
                    break;
                }
            }
    
            if (Pfn1 == EndPfn) {
                RemovedPages = NumberOfPages;
                break;
            }
        }

        InterlockedDecrement (&MiDelayPageFaults);
    }

    if (RemovedPages != NumberOfPages) {
#if DBG
        MiDynmemData[4] += 1;
        if (MiShowStuckPages != 0) {

            RemovedPages = 0;
            for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
                if (Pfn1->u3.e1.PageLocation != BadPageList) {
                    RemovedPages += 1;
                }
            }

            ASSERT (RemovedPages != 0);

            DbgPrint("MmRemovePhysicalMemory : could not get %d of %d pages\n",
                RemovedPages, NumberOfPages);

            if (MiShowStuckPages & 0x2) {

                ULONG PfnsPrinted;
                ULONG EnoughShown;
                PMMPFN FirstPfn;
                PFN_COUNT PfnCount;

                PfnCount = 0;
                PfnsPrinted = 0;
                EnoughShown = 100;
    
                if (MiShowStuckPages & 0x4) {
                    EnoughShown = (ULONG)-1;
                }
    
                DbgPrint("Stuck PFN list: ");
                for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
                    if (Pfn1->u3.e1.PageLocation != BadPageList) {
                        if (PfnCount == 0) {
                            FirstPfn = Pfn1;
                        }
                        PfnCount += 1;
                    }
                    else {
                        if (PfnCount != 0) {
                            DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
                                                    (FirstPfn - MmPfnDatabase) + PfnCount - 1);
                            PfnsPrinted += 1;
                            if (PfnsPrinted == EnoughShown) {
                                break;
                            }
                            PfnCount = 0;
                        }
                    }
                }
                if (PfnCount != 0) {
                    DbgPrint("%x -> %x ; ", FirstPfn - MmPfnDatabase,
                                            (FirstPfn - MmPfnDatabase) + PfnCount - 1);
                }
                DbgPrint("\n");
            }
            if (MiShowStuckPages & 0x8) {
                DbgBreakPoint ();
            }
            if (MiShowStuckPages & 0x10) {
                goto retry;
            }
        }
#endif
        UNLOCK_PFN (OldIrql);
        Status = STATUS_NO_MEMORY;
        goto giveup;
    }

#if DBG
    for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {
        ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
    }
#endif

    //
    // All the pages in the range have been removed.  Update the physical
    // memory blocks and other associated housekeeping.
    //

    if (Additional == 0) {

        //
        // The range can be split off from an end of an existing chunk so no
        // pool growth or shrinkage is required.
        //

        NewPhysicalMemoryBlock = MmPhysicalMemoryBlock;
        OldPhysicalMemoryBlock = NULL;
    }
    else {

        //
        // The range cannot be split off from an end of an existing chunk so
        // pool growth or shrinkage is required.
        //

        UNLOCK_PFN (OldIrql);

        i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
             (sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + Additional)));

        NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
                                                        i,
                                                        '  mM');

        if (NewPhysicalMemoryBlock == NULL) {
            Status = STATUS_INSUFFICIENT_RESOURCES;
#if DBG
            MiDynmemData[5] += 1;
#endif
            goto giveup;
        }

        OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
        RtlZeroMemory (NewPhysicalMemoryBlock, i);

        LOCK_PFN (OldIrql);
    }

    //
    // Remove or split the requested range from the existing memory block.
    //

    NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + Additional;
    NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages - NumberOfPages;

    NewRun = &NewPhysicalMemoryBlock->Run[0];
    start = 0;
    Inserted = FALSE;

    do {

        Page = MmPhysicalMemoryBlock->Run[start].BasePage;
        LastPage = Page + MmPhysicalMemoryBlock->Run[start].PageCount;

        if (Inserted == FALSE) {

            if ((StartPage >= Page) && (EndPage <= LastPage)) {

                if ((StartPage == Page) && (EndPage == LastPage)) {
                    ASSERT (Additional == -1);
                    start += 1;
                    continue;
                }
                else if ((StartPage == Page) || (EndPage == LastPage)) {
                    ASSERT (Additional == 0);
                    if (StartPage == Page) {
                        MmPhysicalMemoryBlock->Run[start].BasePage += NumberOfPages;
                    }
                    MmPhysicalMemoryBlock->Run[start].PageCount -= NumberOfPages;
                }
                else {
                    ASSERT (Additional == 1);

                    OriginalLastPage = LastPage;

                    MmPhysicalMemoryBlock->Run[start].PageCount =
                        StartPage - MmPhysicalMemoryBlock->Run[start].BasePage;

                    *NewRun = MmPhysicalMemoryBlock->Run[start];
                    NewRun += 1;

                    NewRun->BasePage = EndPage;
                    NewRun->PageCount = OriginalLastPage - EndPage;
                    NewRun += 1;

                    start += 1;
                    continue;
                }

                Inserted = TRUE;
            }
        }

        *NewRun = MmPhysicalMemoryBlock->Run[start];
        NewRun += 1;
        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    //
    // Repoint the MmPhysicalMemoryBlock at the new chunk.
    // Free the old block after releasing the PFN lock.
    //

    MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;

    if (EndPage - 1 == MmHighestPhysicalPage) {
        MmHighestPhysicalPage = StartPage - 1;
    }

    //
    // Throw away all the removed pages that are currently enqueued.
    //

    for (Pfn1 = StartPfn; Pfn1 < EndPfn; Pfn1 += 1) {

        ASSERT (Pfn1->u3.e1.PageLocation == BadPageList);
        ASSERT (Pfn1->u3.e1.RemovalRequested == 1);

        MiUnlinkPageFromList (Pfn1);

        ASSERT (Pfn1->u1.Flink == 0);
        ASSERT (Pfn1->u2.Blink == 0);
        ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
        ASSERT64 (Pfn1->UsedPageTableEntries == 0);

        Pfn1->PteAddress = PFN_REMOVED;
        Pfn1->u3.e2.ShortFlags = 0;
        Pfn1->OriginalPte.u.Long = ZeroKernelPte.u.Long;
        Pfn1->PteFrame = 0;
    }

    //
    // Now that the removed pages have been discarded, eliminate the PFN
    // entries that mapped them.  Straddling entries left over from an
    // adjacent earlier removal are not collapsed at this point.
    //
    //

    PagesReleased = 0;

    if (PfnDatabaseIsPhysical == FALSE) {

        VirtualAddress = (PVOID)ROUND_TO_PAGES(MI_PFN_ELEMENT(StartPage));
        PointerPte = MiGetPteAddress (VirtualAddress);
        EndPte = MiGetPteAddress (PAGE_ALIGN(MI_PFN_ELEMENT(EndPage)));

        while (PointerPte < EndPte) {
            PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
            Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
            ASSERT (Pfn1->u2.ShareCount == 1);
            ASSERT (Pfn1->u3.e2.ReferenceCount == 1);
            Pfn1->u2.ShareCount = 0;
            MI_SET_PFN_DELETED (Pfn1);
#if DBG
            Pfn1->u3.e1.PageLocation = StandbyPageList;
#endif //DBG
            MiDecrementReferenceCount (PageFrameIndex);
    
            KeFlushSingleTb (VirtualAddress,
                             TRUE,
                             TRUE,
                             (PHARDWARE_PTE)PointerPte,
                             ZeroKernelPte.u.Flush);
    
            PagesReleased += 1;
            PointerPte += 1;
            VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE);
        }

        MmResidentAvailablePages += PagesReleased;
    }

#if DBG
    MiDynmemData[6] += 1;
#endif

    UNLOCK_PFN (OldIrql);

    if (PagesReleased != 0) {
        MiReturnCommitment (PagesReleased);
    }

    ExReleaseFastMutex (&MmDynamicMemoryMutex);

    if (OldPhysicalMemoryBlock != NULL) {
        ExFreePool (OldPhysicalMemoryBlock);
    }

    NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;

    return STATUS_SUCCESS;

giveup:

    //
    // All the pages in the range were not obtained.  Back everything out.
    //

    PageFrameIndex = StartPage;
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

    LOCK_PFN (OldIrql);

    while (PageFrameIndex < EndPage) {

        ASSERT (Pfn1->u3.e1.RemovalRequested == 1);

        Pfn1->u3.e1.RemovalRequested = 0;

        if ((Pfn1->u3.e1.PageLocation == BadPageList) &&
            (Pfn1->u3.e1.ParityError == 0)) {

            MiUnlinkPageFromList (Pfn1);
            MiInsertPageInList (MmPageLocationList[FreePageList],
                                PageFrameIndex);
        }

        Pfn1 += 1;
        PageFrameIndex += 1;
    }

    MmResidentAvailablePages += NumberOfPages;
    MmNumberOfPhysicalPages += NumberOfPages;

    UNLOCK_PFN (OldIrql);

giveup2:

    ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);
    MmTotalCommitLimit += NumberOfPages;
    MmTotalCommitLimitMaximum += NumberOfPages;
    ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);

    ExReleaseFastMutex (&MmDynamicMemoryMutex);

    return Status;
}
Exemplo n.º 8
0
NTSTATUS
MmAddPhysicalMemory (
    IN PPHYSICAL_ADDRESS StartAddress,
    IN OUT PLARGE_INTEGER NumberOfBytes
    )

/*++

Routine Description:

    This routine adds the specified physical address range to the system.
    This includes initializing PFN database entries and adding it to the
    freelists.

Arguments:

    StartAddress  - Supplies the starting physical address.

    NumberOfBytes  - Supplies a pointer to the number of bytes being added.
                     If any bytes were added (ie: STATUS_SUCCESS is being
                     returned), the actual amount is returned here.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode.  PASSIVE level.  No locks held.

--*/

{
    ULONG i;
    PMMPFN Pfn1;
    KIRQL OldIrql;
    LOGICAL Inserted;
    LOGICAL Updated;
    MMPTE TempPte;
    PMMPTE PointerPte;
    PMMPTE LastPte;
    PFN_NUMBER NumberOfPages;
    PFN_NUMBER start;
    PFN_NUMBER count;
    PFN_NUMBER StartPage;
    PFN_NUMBER EndPage;
    PFN_NUMBER PageFrameIndex;
    PFN_NUMBER Page;
    PFN_NUMBER LastPage;
    PFN_COUNT PagesNeeded;
    PPHYSICAL_MEMORY_DESCRIPTOR OldPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_DESCRIPTOR NewPhysicalMemoryBlock;
    PPHYSICAL_MEMORY_RUN NewRun;
    LOGICAL PfnDatabaseIsPhysical;

    ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL);

    ASSERT (BYTE_OFFSET(NumberOfBytes->LowPart) == 0);
    ASSERT (BYTE_OFFSET(StartAddress->LowPart) == 0);

    if (MI_IS_PHYSICAL_ADDRESS(MmPfnDatabase)) {

        //
        // The system must be configured for dynamic memory addition.  This is
        // critical as only then is the database guaranteed to be non-sparse.
        //
    
        if (MmDynamicPfn == FALSE) {
            return STATUS_NOT_SUPPORTED;
        }

        PfnDatabaseIsPhysical = TRUE;
    }
    else {
        PfnDatabaseIsPhysical = FALSE;
    }

    StartPage = (PFN_NUMBER)(StartAddress->QuadPart >> PAGE_SHIFT);
    NumberOfPages = (PFN_NUMBER)(NumberOfBytes->QuadPart >> PAGE_SHIFT);

    EndPage = StartPage + NumberOfPages;

    if (EndPage - 1 > MmHighestPossiblePhysicalPage) {

        //
        // Truncate the request into something that can be mapped by the PFN
        // database.
        //

        EndPage = MmHighestPossiblePhysicalPage + 1;
        NumberOfPages = EndPage - StartPage;
    }

    //
    // The range cannot wrap.
    //

    if (StartPage >= EndPage) {
        return STATUS_INVALID_PARAMETER_1;
    }

    ExAcquireFastMutex (&MmDynamicMemoryMutex);

    i = (sizeof(PHYSICAL_MEMORY_DESCRIPTOR) +
         (sizeof(PHYSICAL_MEMORY_RUN) * (MmPhysicalMemoryBlock->NumberOfRuns + 1)));

    NewPhysicalMemoryBlock = ExAllocatePoolWithTag (NonPagedPool,
                                                    i,
                                                    '  mM');

    if (NewPhysicalMemoryBlock == NULL) {
        ExReleaseFastMutex (&MmDynamicMemoryMutex);
        return STATUS_INSUFFICIENT_RESOURCES;
    }

    //
    // The range cannot overlap any ranges that are already present.
    //

    start = 0;

    LOCK_PFN (OldIrql);

    do {

        count = MmPhysicalMemoryBlock->Run[start].PageCount;
        Page = MmPhysicalMemoryBlock->Run[start].BasePage;

        if (count != 0) {

            LastPage = Page + count;

            if ((StartPage < Page) && (EndPage > Page)) {
                UNLOCK_PFN (OldIrql);
                ExReleaseFastMutex (&MmDynamicMemoryMutex);
                ExFreePool (NewPhysicalMemoryBlock);
                return STATUS_CONFLICTING_ADDRESSES;
            }

            if ((StartPage >= Page) && (StartPage < LastPage)) {
                UNLOCK_PFN (OldIrql);
                ExReleaseFastMutex (&MmDynamicMemoryMutex);
                ExFreePool (NewPhysicalMemoryBlock);
                return STATUS_CONFLICTING_ADDRESSES;
            }
        }

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    //
    // Fill any gaps in the (sparse) PFN database needed for these pages,
    // unless the PFN database was physically allocated and completely
    // committed up front.
    //

    PagesNeeded = 0;

    if (PfnDatabaseIsPhysical == FALSE) {
        PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
        LastPte = MiGetPteAddress ((PCHAR)(MI_PFN_ELEMENT(EndPage)) - 1);
    
        while (PointerPte <= LastPte) {
            if (PointerPte->u.Hard.Valid == 0) {
                PagesNeeded += 1;
            }
            PointerPte += 1;
        }
    
        if (MmAvailablePages < PagesNeeded) {
            UNLOCK_PFN (OldIrql);
            ExReleaseFastMutex (&MmDynamicMemoryMutex);
            ExFreePool (NewPhysicalMemoryBlock);
            return STATUS_INSUFFICIENT_RESOURCES;
        }
    
        TempPte = ValidKernelPte;
    
        PointerPte = MiGetPteAddress (MI_PFN_ELEMENT(StartPage));
    
        while (PointerPte <= LastPte) {
            if (PointerPte->u.Hard.Valid == 0) {
    
                PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
    
                MiInitializePfn (PageFrameIndex, PointerPte, 0);
    
                TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
                *PointerPte = TempPte;
            }
            PointerPte += 1;
        }
        MmResidentAvailablePages -= PagesNeeded;
    }

    //
    // If the new range is adjacent to an existing range, just merge it into
    // the old block.  Otherwise use the new block as a new entry will have to
    // be used.
    //

    NewPhysicalMemoryBlock->NumberOfRuns = MmPhysicalMemoryBlock->NumberOfRuns + 1;
    NewPhysicalMemoryBlock->NumberOfPages = MmPhysicalMemoryBlock->NumberOfPages + NumberOfPages;

    NewRun = &NewPhysicalMemoryBlock->Run[0];
    start = 0;
    Inserted = FALSE;
    Updated = FALSE;

    do {

        Page = MmPhysicalMemoryBlock->Run[start].BasePage;
        count = MmPhysicalMemoryBlock->Run[start].PageCount;

        if (Inserted == FALSE) {

            //
            // Note overlaps into adjacent ranges were already checked above.
            //

            if (StartPage == Page + count) {
                MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
                OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
                MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;

                //
                // Coalesce below and above to avoid leaving zero length gaps
                // as these gaps would prevent callers from removing ranges
                // the span them.
                //

                if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {

                    start += 1;
                    Page = MmPhysicalMemoryBlock->Run[start].BasePage;
                    count = MmPhysicalMemoryBlock->Run[start].PageCount;

                    if (StartPage + NumberOfPages == Page) {
                        MmPhysicalMemoryBlock->Run[start - 1].PageCount +=
                            count;
                        MmPhysicalMemoryBlock->NumberOfRuns -= 1;

                        //
                        // Copy any remaining entries.
                        //
    
                        if (start != MmPhysicalMemoryBlock->NumberOfRuns) {
                            RtlMoveMemory (&MmPhysicalMemoryBlock->Run[start],
                                           &MmPhysicalMemoryBlock->Run[start + 1],
                                           (MmPhysicalMemoryBlock->NumberOfRuns - start) * sizeof (PHYSICAL_MEMORY_RUN));
                        }
                    }
                }
                Updated = TRUE;
                break;
            }

            if (StartPage + NumberOfPages == Page) {
                MmPhysicalMemoryBlock->Run[start].BasePage = StartPage;
                MmPhysicalMemoryBlock->Run[start].PageCount += NumberOfPages;
                OldPhysicalMemoryBlock = NewPhysicalMemoryBlock;
                MmPhysicalMemoryBlock->NumberOfPages += NumberOfPages;
                Updated = TRUE;
                break;
            }

            if (StartPage + NumberOfPages <= Page) {

                if (start + 1 < MmPhysicalMemoryBlock->NumberOfRuns) {

                    if (StartPage + NumberOfPages <= MmPhysicalMemoryBlock->Run[start + 1].BasePage) {
                        //
                        // Don't insert here - the new entry really belongs
                        // (at least) one entry further down.
                        //

                        continue;
                    }
                }

                NewRun->BasePage = StartPage;
                NewRun->PageCount = NumberOfPages;
                NewRun += 1;
                Inserted = TRUE;
                Updated = TRUE;
            }
        }

        *NewRun = MmPhysicalMemoryBlock->Run[start];
        NewRun += 1;

        start += 1;

    } while (start != MmPhysicalMemoryBlock->NumberOfRuns);

    //
    // If the memory block has not been updated, then the new entry must
    // be added at the very end.
    //

    if (Updated == FALSE) {
        ASSERT (Inserted == FALSE);
        NewRun->BasePage = StartPage;
        NewRun->PageCount = NumberOfPages;
        Inserted = TRUE;
    }

    //
    // Repoint the MmPhysicalMemoryBlock at the new chunk, free the old after
    // releasing the PFN lock.
    //

    if (Inserted == TRUE) {
        OldPhysicalMemoryBlock = MmPhysicalMemoryBlock;
        MmPhysicalMemoryBlock = NewPhysicalMemoryBlock;
    }

    //
    // Note that the page directory (page parent entries on Win64) must be
    // filled in at system boot so that already-created processes do not fault
    // when referencing the new PFNs.
    //

    //
    // Walk through the memory descriptors and add pages to the
    // free list in the PFN database.
    //

    PageFrameIndex = StartPage;
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

    if (EndPage - 1 > MmHighestPhysicalPage) {
        MmHighestPhysicalPage = EndPage - 1;
    }

    while (PageFrameIndex < EndPage) {

        ASSERT (Pfn1->u2.ShareCount == 0);
        ASSERT (Pfn1->u3.e2.ShortFlags == 0);
        ASSERT (Pfn1->u3.e2.ReferenceCount == 0);
        ASSERT64 (Pfn1->UsedPageTableEntries == 0);
        ASSERT (Pfn1->OriginalPte.u.Long == ZeroKernelPte.u.Long);
        ASSERT (Pfn1->PteFrame == 0);
        ASSERT ((Pfn1->PteAddress == PFN_REMOVED) ||
                (Pfn1->PteAddress == (PMMPTE)(UINT_PTR)0));

        //
        // Set the PTE address to the physical page for
        // virtual address alignment checking.
        //

        Pfn1->PteAddress = (PMMPTE)(PageFrameIndex << PTE_SHIFT);

        MiInsertPageInList (MmPageLocationList[FreePageList],
                            PageFrameIndex);

        PageFrameIndex += 1;
        
        Pfn1 += 1;
    }

    MmResidentAvailablePages += NumberOfPages;
    MmNumberOfPhysicalPages += (PFN_COUNT)NumberOfPages;

    UNLOCK_PFN (OldIrql);

    //
    // Increase all commit limits to reflect the additional memory.
    //

    ExAcquireSpinLock (&MmChargeCommitmentLock, &OldIrql);

    MmTotalCommitLimit += NumberOfPages;
    MmTotalCommitLimitMaximum += NumberOfPages;

    MmTotalCommittedPages += PagesNeeded;

    ExReleaseSpinLock (&MmChargeCommitmentLock, OldIrql);

    ExReleaseFastMutex (&MmDynamicMemoryMutex);

    ExFreePool (OldPhysicalMemoryBlock);

    //
    // Indicate number of bytes actually added to our caller.
    //

    NumberOfBytes->QuadPart = (ULONGLONG)NumberOfPages * PAGE_SIZE;

    return STATUS_SUCCESS;
}