/* * @implemented */ BOOLEAN NTAPI MmIsAddressValid(IN PVOID VirtualAddress) { #if _MI_PAGING_LEVELS >= 4 /* Check if the PXE is valid */ if (MiAddressToPxe(VirtualAddress)->u.Hard.Valid == 0) return FALSE; #endif #if _MI_PAGING_LEVELS >= 3 /* Check if the PPE is valid */ if (MiAddressToPpe(VirtualAddress)->u.Hard.Valid == 0) return FALSE; #endif #if _MI_PAGING_LEVELS >= 2 /* Check if the PDE is valid */ if (MiAddressToPde(VirtualAddress)->u.Hard.Valid == 0) return FALSE; #endif /* Check if the PTE is valid */ if (MiAddressToPte(VirtualAddress)->u.Hard.Valid == 0) return FALSE; /* This address is valid now, but it will only stay so if the caller holds * the PFN lock */ return TRUE; }
EXTERN_C static bool Win8pIsAccessibleAddress( __in void* Address) { PAGED_CODE(); const auto pxe = MiAddressToPxe(Address); const auto ppe = MiAddressToPpe(Address); const auto pde = MiAddressToPde(Address); const auto pte = MiAddressToPte(Address); if ((!pxe->Valid) || (!ppe->Valid) || (!pde->Valid) || (!pde->LargePage && (!pte || !pte->Valid))) { return false; } return true; }
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes) { PFN_NUMBER PageFrameNumber; PFN_COUNT SizeInPages, PageTableCount; ULONG i; KIRQL OldIrql; PLIST_ENTRY NextEntry, NextHead, LastHead; PMMPTE PointerPte, StartPte; PMMPDE PointerPde; ULONG EndAllocation; MMPTE TempPte; MMPDE TempPde; PMMPFN Pfn1; PVOID BaseVa, BaseVaStart; PMMFREE_POOL_ENTRY FreeEntry; PKSPIN_LOCK_QUEUE LockQueue; // // Figure out how big the allocation is in pages // SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes); // // Check for overflow // if (SizeInPages == 0) { // // Fail // return NULL; } // // Handle paged pool // if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Lock the paged pool mutex // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Find some empty allocation space // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, MmPagedPoolInfo.PagedPoolHint); if (i == 0xFFFFFFFF) { // // Get the page bit count // i = ((SizeInPages - 1) / PTE_COUNT) + 1; DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages); // // Check if there is enougn paged pool expansion space left // if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } // // Check if we'll have to expand past the last PTE we have available // if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // We can only support this much then // PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool); PageTableCount = (PFN_COUNT)(PointerPde + 1 - MmPagedPoolInfo.NextPdeForPagedPoolExpansion); ASSERT(PageTableCount < i); i = PageTableCount; } else { // // Otherwise, there is plenty of space left for this expansion // PageTableCount = i; } // // Get the template PDE we'll use to expand // TempPde = ValidKernelPde; // // Get the first PTE in expansion space // PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion; BaseVa = MiPdeToPte(PointerPde); BaseVaStart = BaseVa; // // Lock the PFN database and loop pages // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // It should not already be valid // ASSERT(PointerPde->u.Hard.Valid == 0); /* Request a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPde.u.Hard.PageFrameNumber = PageFrameNumber; #if (_MI_PAGING_LEVELS >= 3) /* On PAE/x64 systems, there's no double-buffering */ ASSERT(FALSE); #else // // Save it into our double-buffered system page directory // MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde; /* Initialize the PFN */ MiInitializePfnForOtherProcess(PageFrameNumber, (PMMPTE)PointerPde, MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]); /* Write the actual PDE now */ // MI_WRITE_VALID_PDE(PointerPde, TempPde); #endif // // Move on to the next expansion address // PointerPde++; BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE); i--; } while (i > 0); // // Release the PFN database lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // These pages are now available, clear their availablity bits // EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion - (PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) * PTE_COUNT; RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, EndAllocation, PageTableCount * PTE_COUNT); // // Update the next expansion location // MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount; // // Zero out the newly available memory // RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE); // // Now try consuming the pages again // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, 0); if (i == 0xFFFFFFFF) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } } // // Update the pool hint if the request was just one page // if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1; // // Update the end bitmap so we know the bounds of this allocation when // the time comes to free it // EndAllocation = i + SizeInPages - 1; RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation); // // Now we can release the lock (it mainly protects the bitmap) // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // Now figure out where this allocation starts // BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT)); // // Flush the TLB // KeFlushEntireTb(TRUE, TRUE); /* Setup a demand-zero writable PTE */ MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE); // // Find the first and last PTE, then loop them all // PointerPte = MiAddressToPte(BaseVa); StartPte = PointerPte + SizeInPages; do { // // Write the demand zero PTE and keep going // MI_WRITE_INVALID_PTE(PointerPte, TempPte); } while (++PointerPte < StartPte); // // Return the allocation address to the caller // return BaseVa; } // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Allocations of less than 4 pages go into their individual buckets // i = SizeInPages - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; // // Loop through all the free page lists based on the page index // NextHead = &MmNonPagedPoolFreeListHead[i]; LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS]; // // Acquire the nonpaged pool lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); do { // // Now loop through all the free page entries in this given list // NextEntry = NextHead->Flink; while (NextEntry != NextHead) { /* Is freed non paged pool enabled */ if (MmProtectFreedNonPagedPool) { /* We need to be able to touch this page, unprotect it */ MiUnProtectFreeNonPagedPool(NextEntry, 0); } // // Grab the entry and see if it can handle our allocation // FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List); ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE); if (FreeEntry->Size >= SizeInPages) { // // It does, so consume the pages from here // FreeEntry->Size -= SizeInPages; // // The allocation will begin in this free page area // BaseVa = (PVOID)((ULONG_PTR)FreeEntry + (FreeEntry->Size << PAGE_SHIFT)); /* Remove the item from the list, depending if pool is protected */ if (MmProtectFreedNonPagedPool) MiProtectedPoolRemoveEntryList(&FreeEntry->List); else RemoveEntryList(&FreeEntry->List); // // However, check if its' still got space left // if (FreeEntry->Size != 0) { /* Check which list to insert this entry into */ i = FreeEntry->Size - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; /* Insert the entry into the free list head, check for prot. pool */ if (MmProtectFreedNonPagedPool) MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE); else InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List); /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } // // Grab the PTE for this allocation // PointerPte = MiAddressToPte(BaseVa); ASSERT(PointerPte->u.Hard.Valid == 1); // // Grab the PFN NextEntry and index // Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); // // Now mark it as the beginning of an allocation // ASSERT(Pfn1->u3.e1.StartOfAllocation == 0); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as special pool if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) { Pfn1->u4.VerifierAllocation = 1; } // // Check if the allocation is larger than one page // if (SizeInPages != 1) { // // Navigate to the last PFN entry and PTE // PointerPte += SizeInPages - 1; ASSERT(PointerPte->u.Hard.Valid == 1); Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber); } // // Mark this PFN as the last (might be the same as the first) // ASSERT(Pfn1->u3.e1.EndOfAllocation == 0); Pfn1->u3.e1.EndOfAllocation = 1; // // Release the nonpaged pool lock, and return the allocation // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); return BaseVa; } // // Try the next free page entry // NextEntry = FreeEntry->List.Flink; /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } } while (++NextHead < LastHead); // // If we got here, we're out of space. // Start by releasing the lock // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Allocate some system PTEs // StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion); PointerPte = StartPte; if (StartPte == NULL) { // // Ran out of memory // DPRINT1("Out of NP Expansion Pool\n"); return NULL; } // // Acquire the pool lock now // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); // // Lock the PFN database too // LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock]; KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); // // Loop the pages // TempPte = ValidKernelPte; do { /* Allocate a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->PteAddress = PointerPte; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u4.VerifierAllocation = 0; /* Write the PTE for it */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MI_WRITE_VALID_PTE(PointerPte++, TempPte); } while (--SizeInPages > 0); // // This is the last page // Pfn1->u3.e1.EndOfAllocation = 1; // // Get the first page and mark it as such // Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as a verifier allocation if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1; // // Release the PFN and nonpaged pool lock // KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Return the address // return MiPteToAddress(StartPte); }
/* * @implemented */ VOID NTAPI MmProbeAndLockPages(IN PMDL Mdl, IN KPROCESSOR_MODE AccessMode, IN LOCK_OPERATION Operation) { PPFN_NUMBER MdlPages; PVOID Base, Address, LastAddress, StartAddress; ULONG LockPages, TotalPages; NTSTATUS Status = STATUS_SUCCESS; PEPROCESS CurrentProcess; NTSTATUS ProbeStatus; PMMPTE PointerPte, LastPte; PMMPDE PointerPde; #if (_MI_PAGING_LEVELS >= 3) PMMPDE PointerPpe; #endif #if (_MI_PAGING_LEVELS == 4) PMMPDE PointerPxe; #endif PFN_NUMBER PageFrameIndex; BOOLEAN UsePfnLock; KIRQL OldIrql; PMMPFN Pfn1; DPRINT("Probing MDL: %p\n", Mdl); // // Sanity checks // ASSERT(Mdl->ByteCount != 0); ASSERT(((ULONG)Mdl->ByteOffset & ~(PAGE_SIZE - 1)) == 0); ASSERT(((ULONG_PTR)Mdl->StartVa & (PAGE_SIZE - 1)) == 0); ASSERT((Mdl->MdlFlags & (MDL_PAGES_LOCKED | MDL_MAPPED_TO_SYSTEM_VA | MDL_SOURCE_IS_NONPAGED_POOL | MDL_PARTIAL | MDL_IO_SPACE)) == 0); // // Get page and base information // MdlPages = (PPFN_NUMBER)(Mdl + 1); Base = Mdl->StartVa; // // Get the addresses and how many pages we span (and need to lock) // Address = (PVOID)((ULONG_PTR)Base + Mdl->ByteOffset); LastAddress = (PVOID)((ULONG_PTR)Address + Mdl->ByteCount); LockPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Address, Mdl->ByteCount); ASSERT(LockPages != 0); /* Block invalid access */ if ((AccessMode != KernelMode) && ((LastAddress > (PVOID)MM_USER_PROBE_ADDRESS) || (Address >= LastAddress))) { /* Caller should be in SEH, raise the error */ *MdlPages = LIST_HEAD; ExRaiseStatus(STATUS_ACCESS_VIOLATION); } // // Get the process // if (Address <= MM_HIGHEST_USER_ADDRESS) { // // Get the process // CurrentProcess = PsGetCurrentProcess(); } else { // // No process // CurrentProcess = NULL; } // // Save the number of pages we'll have to lock, and the start address // TotalPages = LockPages; StartAddress = Address; /* Large pages not supported */ ASSERT(!MI_IS_PHYSICAL_ADDRESS(Address)); // // Now probe them // ProbeStatus = STATUS_SUCCESS; _SEH2_TRY { // // Enter probe loop // do { // // Assume failure // *MdlPages = LIST_HEAD; // // Read // *(volatile CHAR*)Address; // // Check if this is write access (only probe for user-mode) // if ((Operation != IoReadAccess) && (Address <= MM_HIGHEST_USER_ADDRESS)) { // // Probe for write too // ProbeForWriteChar(Address); } // // Next address... // Address = PAGE_ALIGN((ULONG_PTR)Address + PAGE_SIZE); // // Next page... // LockPages--; MdlPages++; } while (Address < LastAddress); // // Reset back to the original page // ASSERT(LockPages == 0); MdlPages = (PPFN_NUMBER)(Mdl + 1); } _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { // // Oops :( // ProbeStatus = _SEH2_GetExceptionCode(); } _SEH2_END; // // So how did that go? // if (ProbeStatus != STATUS_SUCCESS) { // // Fail // DPRINT1("MDL PROBE FAILED!\n"); Mdl->Process = NULL; ExRaiseStatus(ProbeStatus); } // // Get the PTE and PDE // PointerPte = MiAddressToPte(StartAddress); PointerPde = MiAddressToPde(StartAddress); #if (_MI_PAGING_LEVELS >= 3) PointerPpe = MiAddressToPpe(StartAddress); #endif #if (_MI_PAGING_LEVELS == 4) PointerPxe = MiAddressToPxe(StartAddress); #endif // // Sanity check // ASSERT(MdlPages == (PPFN_NUMBER)(Mdl + 1)); // // Check what kind of operation this is // if (Operation != IoReadAccess) { // // Set the write flag // Mdl->MdlFlags |= MDL_WRITE_OPERATION; } else { // // Remove the write flag // Mdl->MdlFlags &= ~(MDL_WRITE_OPERATION); } // // Mark the MDL as locked *now* // Mdl->MdlFlags |= MDL_PAGES_LOCKED; // // Check if this came from kernel mode // if (Base > MM_HIGHEST_USER_ADDRESS) { // // We should not have a process // ASSERT(CurrentProcess == NULL); Mdl->Process = NULL; // // In kernel mode, we don't need to check for write access // Operation = IoReadAccess; // // Use the PFN lock // UsePfnLock = TRUE; OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } else { // // Sanity checks // ASSERT(TotalPages != 0); ASSERT(CurrentProcess == PsGetCurrentProcess()); // // Track locked pages // InterlockedExchangeAddSizeT(&CurrentProcess->NumberOfLockedPages, TotalPages); // // Save the process // Mdl->Process = CurrentProcess; /* Lock the process working set */ MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); UsePfnLock = FALSE; OldIrql = MM_NOIRQL; } // // Get the last PTE // LastPte = MiAddressToPte((PVOID)((ULONG_PTR)LastAddress - 1)); // // Loop the pages // do { // // Assume failure and check for non-mapped pages // *MdlPages = LIST_HEAD; while ( #if (_MI_PAGING_LEVELS == 4) (PointerPxe->u.Hard.Valid == 0) || #endif #if (_MI_PAGING_LEVELS >= 3) (PointerPpe->u.Hard.Valid == 0) || #endif (PointerPde->u.Hard.Valid == 0) || (PointerPte->u.Hard.Valid == 0)) { // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Access the page // Address = MiPteToAddress(PointerPte); //HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked Status = MmAccessFault(FALSE, Address, KernelMode, (PVOID)0xBADBADA3); if (!NT_SUCCESS(Status)) { // // Fail // DPRINT1("Access fault failed\n"); goto Cleanup; } // // What lock should we use? // if (UsePfnLock) { // // Grab the PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } else { /* Lock the process working set */ MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } } // // Check if this was a write or modify // if (Operation != IoReadAccess) { // // Check if the PTE is not writable // if (MI_IS_PAGE_WRITEABLE(PointerPte) == FALSE) { // // Check if it's copy on write // if (MI_IS_PAGE_COPY_ON_WRITE(PointerPte)) { // // Get the base address and allow a change for user-mode // Address = MiPteToAddress(PointerPte); if (Address <= MM_HIGHEST_USER_ADDRESS) { // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Access the page // //HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked Status = MmAccessFault(TRUE, Address, KernelMode, (PVOID)0xBADBADA3); if (!NT_SUCCESS(Status)) { // // Fail // DPRINT1("Access fault failed\n"); goto Cleanup; } // // Re-acquire the lock // if (UsePfnLock) { // // Grab the PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } else { /* Lock the process working set */ MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Start over // continue; } } // // Fail, since we won't allow this // Status = STATUS_ACCESS_VIOLATION; goto CleanupWithLock; } } // // Grab the PFN // PageFrameIndex = PFN_FROM_PTE(PointerPte); Pfn1 = MiGetPfnEntry(PageFrameIndex); if (Pfn1) { /* Either this is for kernel-mode, or the working set is held */ ASSERT((CurrentProcess == NULL) || (UsePfnLock == FALSE)); /* No Physical VADs supported yet */ if (CurrentProcess) ASSERT(CurrentProcess->PhysicalVadRoot == NULL); /* This address should already exist and be fully valid */ MiReferenceProbedPageAndBumpLockCount(Pfn1); } else { // // For I/O addresses, just remember this // Mdl->MdlFlags |= MDL_IO_SPACE; } // // Write the page and move on // *MdlPages++ = PageFrameIndex; PointerPte++; /* Check if we're on a PDE boundary */ if (MiIsPteOnPdeBoundary(PointerPte)) PointerPde++; #if (_MI_PAGING_LEVELS >= 3) if (MiIsPteOnPpeBoundary(PointerPte)) PointerPpe++; #endif #if (_MI_PAGING_LEVELS == 4) if (MiIsPteOnPxeBoundary(PointerPte)) PointerPxe++; #endif } while (PointerPte <= LastPte); // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Sanity check // ASSERT((Mdl->MdlFlags & MDL_DESCRIBES_AWE) == 0); return; CleanupWithLock: // // This is the failure path // ASSERT(!NT_SUCCESS(Status)); // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } Cleanup: // // Pages must be locked so MmUnlock can work // ASSERT(Mdl->MdlFlags & MDL_PAGES_LOCKED); MmUnlockPages(Mdl); // // Raise the error // ExRaiseStatus(Status); }
static PMMPTE MiGetPteForProcess( PEPROCESS Process, PVOID Address, BOOLEAN Create) { MMPTE TmplPte, *Pte; /* Check if we need hypersapce mapping */ if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) { UNIMPLEMENTED; __debugbreak(); return NULL; } else if (Create) { KIRQL OldIrql; TmplPte.u.Long = 0; TmplPte.u.Flush.Valid = 1; TmplPte.u.Flush.Write = 1; /* All page table levels of user pages are user owned */ TmplPte.u.Flush.Owner = (Address < MmHighestUserAddress) ? 1 : 0; /* Lock the PFN database */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Get the PXE */ Pte = MiAddressToPxe(Address); if (!Pte->u.Hard.Valid) { TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(0); MI_WRITE_VALID_PTE(Pte, TmplPte); } /* Get the PPE */ Pte = MiAddressToPpe(Address); if (!Pte->u.Hard.Valid) { TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(1); MI_WRITE_VALID_PTE(Pte, TmplPte); } /* Get the PDE */ Pte = MiAddressToPde(Address); if (!Pte->u.Hard.Valid) { TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(2); MI_WRITE_VALID_PTE(Pte, TmplPte); } /* Unlock PFN database */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Get the PXE */ Pte = MiAddressToPxe(Address); if (!Pte->u.Hard.Valid) return NULL; /* Get the PPE */ Pte = MiAddressToPpe(Address); if (!Pte->u.Hard.Valid) return NULL; /* Get the PDE */ Pte = MiAddressToPde(Address); if (!Pte->u.Hard.Valid) return NULL; } return MiAddressToPte(Address); }
VOID NTAPI MiBalancerThread(PVOID Unused) { PVOID WaitObjects[2]; NTSTATUS Status; ULONG i; WaitObjects[0] = &MiBalancerEvent; WaitObjects[1] = &MiBalancerTimer; while (1) { Status = KeWaitForMultipleObjects(2, WaitObjects, WaitAny, Executive, KernelMode, FALSE, NULL, NULL); if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1) { ULONG InitialTarget = 0; #if (_MI_PAGING_LEVELS == 2) if (!MiIsBalancerThread()) { /* Clean up the unused PDEs */ ULONG_PTR Address; PEPROCESS Process = PsGetCurrentProcess(); /* Acquire PFN lock */ KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); PMMPDE pointerPde; for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS; Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS; Address += (PAGE_SIZE * PTE_COUNT)) { if (MiQueryPageTableReferences((PVOID)Address) == 0) { pointerPde = MiAddressToPde(Address); if (pointerPde->u.Hard.Valid) MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL); ASSERT(pointerPde->u.Hard.Valid == 0); } } /* Release lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } #endif do { ULONG OldTarget = InitialTarget; /* Trim each consumer */ for (i = 0; i < MC_MAXIMUM; i++) { InitialTarget = MiTrimMemoryConsumer(i, InitialTarget); } /* No pages left to swap! */ if (InitialTarget != 0 && InitialTarget == OldTarget) { /* Game over */ KeBugCheck(NO_PAGES_AVAILABLE); } } while (InitialTarget != 0); } else { DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status); KeBugCheck(MEMORY_MANAGEMENT); } } }