PFN_NUMBER NTAPI MiFindContiguousPages(IN PFN_NUMBER LowestPfn, IN PFN_NUMBER HighestPfn, IN PFN_NUMBER BoundaryPfn, IN PFN_NUMBER SizeInPages, IN MEMORY_CACHING_TYPE CacheType) { PFN_NUMBER Page, PageCount, LastPage, Length, BoundaryMask; ULONG i = 0; PMMPFN Pfn1, EndPfn; KIRQL OldIrql; PAGED_CODE(); ASSERT(SizeInPages != 0); // // Convert the boundary PFN into an alignment mask // BoundaryMask = ~(BoundaryPfn - 1); /* Disable APCs */ KeEnterGuardedRegion(); // // Loop all the physical memory blocks // do { // // Capture the base page and length of this memory block // Page = MmPhysicalMemoryBlock->Run[i].BasePage; PageCount = MmPhysicalMemoryBlock->Run[i].PageCount; // // Check how far this memory block will go // LastPage = Page + PageCount; // // Trim it down to only the PFNs we're actually interested in // if ((LastPage - 1) > HighestPfn) LastPage = HighestPfn + 1; if (Page < LowestPfn) Page = LowestPfn; // // Skip this run if it's empty or fails to contain all the pages we need // if (!(PageCount) || ((Page + SizeInPages) > LastPage)) continue; // // Now scan all the relevant PFNs in this run // Length = 0; for (Pfn1 = MI_PFN_ELEMENT(Page); Page < LastPage; Page++, Pfn1++) { // // If this PFN is in use, ignore it // if (MiIsPfnInUse(Pfn1)) { Length = 0; continue; } // // If we haven't chosen a start PFN yet and the caller specified an // alignment, make sure the page matches the alignment restriction // if ((!(Length) && (BoundaryPfn)) && (((Page ^ (Page + SizeInPages - 1)) & BoundaryMask))) { // // It does not, so bail out // continue; } // // Increase the number of valid pages, and check if we have enough // if (++Length == SizeInPages) { // // It appears we've amassed enough legitimate pages, rollback // Pfn1 -= (Length - 1); Page -= (Length - 1); // // Acquire the PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // Things might've changed for us. Is the page still free? // if (MiIsPfnInUse(Pfn1)) break; // // So far so good. Is this the last confirmed valid page? // if (!--Length) { // // Sanity check that we didn't go out of bounds // ASSERT(i != MmPhysicalMemoryBlock->NumberOfRuns); // // Loop until all PFN entries have been processed // EndPfn = Pfn1 - SizeInPages + 1; do { // // This PFN is now a used page, set it up // MI_SET_USAGE(MI_USAGE_CONTINOUS_ALLOCATION); MI_SET_PROCESS2("Kernel Driver"); MiUnlinkFreeOrZeroedPage(Pfn1); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u3.e1.StartOfAllocation = 0; Pfn1->u3.e1.EndOfAllocation = 0; Pfn1->u3.e1.PrototypePte = 0; Pfn1->u4.VerifierAllocation = 0; Pfn1->PteAddress = (PVOID)0xBAADF00D; // // Check if this is the last PFN, otherwise go on // if (Pfn1 == EndPfn) break; Pfn1--; } while (TRUE); // // Mark the first and last PFN so we can find them later // Pfn1->u3.e1.StartOfAllocation = 1; (Pfn1 + SizeInPages - 1)->u3.e1.EndOfAllocation = 1; // // Now it's safe to let go of the PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // Quick sanity check that the last PFN is consistent // EndPfn = Pfn1 + SizeInPages; ASSERT(EndPfn == MI_PFN_ELEMENT(Page + 1)); // // Compute the first page, and make sure it's consistent // Page = Page - SizeInPages + 1; ASSERT(Pfn1 == MI_PFN_ELEMENT(Page)); ASSERT(Page != 0); /* Enable APCs and return the page */ KeLeaveGuardedRegion(); return Page; } // // Keep going. The purpose of this loop is to reconfirm that // after acquiring the PFN lock these pages are still usable // Pfn1++; Page++; } while (TRUE); // // If we got here, something changed while we hadn't acquired // the PFN lock yet, so we'll have to restart // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); Length = 0; } } } while (++i != MmPhysicalMemoryBlock->NumberOfRuns); // // And if we get here, it means no suitable physical memory runs were found // return 0; }
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes) { PFN_NUMBER PageFrameNumber; PFN_COUNT SizeInPages, PageTableCount; ULONG i; KIRQL OldIrql; PLIST_ENTRY NextEntry, NextHead, LastHead; PMMPTE PointerPte, StartPte; PMMPDE PointerPde; ULONG EndAllocation; MMPTE TempPte; MMPDE TempPde; PMMPFN Pfn1; PVOID BaseVa, BaseVaStart; PMMFREE_POOL_ENTRY FreeEntry; PKSPIN_LOCK_QUEUE LockQueue; // // Figure out how big the allocation is in pages // SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes); // // Check for overflow // if (SizeInPages == 0) { // // Fail // return NULL; } // // Handle paged pool // if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Lock the paged pool mutex // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Find some empty allocation space // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, MmPagedPoolInfo.PagedPoolHint); if (i == 0xFFFFFFFF) { // // Get the page bit count // i = ((SizeInPages - 1) / PTE_COUNT) + 1; DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages); // // Check if there is enougn paged pool expansion space left // if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } // // Check if we'll have to expand past the last PTE we have available // if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // We can only support this much then // PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool); PageTableCount = (PFN_COUNT)(PointerPde + 1 - MmPagedPoolInfo.NextPdeForPagedPoolExpansion); ASSERT(PageTableCount < i); i = PageTableCount; } else { // // Otherwise, there is plenty of space left for this expansion // PageTableCount = i; } // // Get the template PDE we'll use to expand // TempPde = ValidKernelPde; // // Get the first PTE in expansion space // PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion; BaseVa = MiPdeToPte(PointerPde); BaseVaStart = BaseVa; // // Lock the PFN database and loop pages // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // It should not already be valid // ASSERT(PointerPde->u.Hard.Valid == 0); /* Request a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPde.u.Hard.PageFrameNumber = PageFrameNumber; #if (_MI_PAGING_LEVELS >= 3) /* On PAE/x64 systems, there's no double-buffering */ ASSERT(FALSE); #else // // Save it into our double-buffered system page directory // MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde; /* Initialize the PFN */ MiInitializePfnForOtherProcess(PageFrameNumber, (PMMPTE)PointerPde, MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]); /* Write the actual PDE now */ // MI_WRITE_VALID_PDE(PointerPde, TempPde); #endif // // Move on to the next expansion address // PointerPde++; BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE); i--; } while (i > 0); // // Release the PFN database lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // These pages are now available, clear their availablity bits // EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion - (PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) * PTE_COUNT; RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, EndAllocation, PageTableCount * PTE_COUNT); // // Update the next expansion location // MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount; // // Zero out the newly available memory // RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE); // // Now try consuming the pages again // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, 0); if (i == 0xFFFFFFFF) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } } // // Update the pool hint if the request was just one page // if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1; // // Update the end bitmap so we know the bounds of this allocation when // the time comes to free it // EndAllocation = i + SizeInPages - 1; RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation); // // Now we can release the lock (it mainly protects the bitmap) // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // Now figure out where this allocation starts // BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT)); // // Flush the TLB // KeFlushEntireTb(TRUE, TRUE); /* Setup a demand-zero writable PTE */ MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE); // // Find the first and last PTE, then loop them all // PointerPte = MiAddressToPte(BaseVa); StartPte = PointerPte + SizeInPages; do { // // Write the demand zero PTE and keep going // MI_WRITE_INVALID_PTE(PointerPte, TempPte); } while (++PointerPte < StartPte); // // Return the allocation address to the caller // return BaseVa; } // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Allocations of less than 4 pages go into their individual buckets // i = SizeInPages - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; // // Loop through all the free page lists based on the page index // NextHead = &MmNonPagedPoolFreeListHead[i]; LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS]; // // Acquire the nonpaged pool lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); do { // // Now loop through all the free page entries in this given list // NextEntry = NextHead->Flink; while (NextEntry != NextHead) { /* Is freed non paged pool enabled */ if (MmProtectFreedNonPagedPool) { /* We need to be able to touch this page, unprotect it */ MiUnProtectFreeNonPagedPool(NextEntry, 0); } // // Grab the entry and see if it can handle our allocation // FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List); ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE); if (FreeEntry->Size >= SizeInPages) { // // It does, so consume the pages from here // FreeEntry->Size -= SizeInPages; // // The allocation will begin in this free page area // BaseVa = (PVOID)((ULONG_PTR)FreeEntry + (FreeEntry->Size << PAGE_SHIFT)); /* Remove the item from the list, depending if pool is protected */ if (MmProtectFreedNonPagedPool) MiProtectedPoolRemoveEntryList(&FreeEntry->List); else RemoveEntryList(&FreeEntry->List); // // However, check if its' still got space left // if (FreeEntry->Size != 0) { /* Check which list to insert this entry into */ i = FreeEntry->Size - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; /* Insert the entry into the free list head, check for prot. pool */ if (MmProtectFreedNonPagedPool) MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE); else InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List); /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } // // Grab the PTE for this allocation // PointerPte = MiAddressToPte(BaseVa); ASSERT(PointerPte->u.Hard.Valid == 1); // // Grab the PFN NextEntry and index // Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); // // Now mark it as the beginning of an allocation // ASSERT(Pfn1->u3.e1.StartOfAllocation == 0); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as special pool if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) { Pfn1->u4.VerifierAllocation = 1; } // // Check if the allocation is larger than one page // if (SizeInPages != 1) { // // Navigate to the last PFN entry and PTE // PointerPte += SizeInPages - 1; ASSERT(PointerPte->u.Hard.Valid == 1); Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber); } // // Mark this PFN as the last (might be the same as the first) // ASSERT(Pfn1->u3.e1.EndOfAllocation == 0); Pfn1->u3.e1.EndOfAllocation = 1; // // Release the nonpaged pool lock, and return the allocation // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); return BaseVa; } // // Try the next free page entry // NextEntry = FreeEntry->List.Flink; /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } } while (++NextHead < LastHead); // // If we got here, we're out of space. // Start by releasing the lock // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Allocate some system PTEs // StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion); PointerPte = StartPte; if (StartPte == NULL) { // // Ran out of memory // DPRINT1("Out of NP Expansion Pool\n"); return NULL; } // // Acquire the pool lock now // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); // // Lock the PFN database too // LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock]; KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); // // Loop the pages // TempPte = ValidKernelPte; do { /* Allocate a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->PteAddress = PointerPte; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u4.VerifierAllocation = 0; /* Write the PTE for it */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MI_WRITE_VALID_PTE(PointerPte++, TempPte); } while (--SizeInPages > 0); // // This is the last page // Pfn1->u3.e1.EndOfAllocation = 1; // // Get the first page and mark it as such // Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as a verifier allocation if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1; // // Release the PFN and nonpaged pool lock // KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Return the address // return MiPteToAddress(StartPte); }
VOID NTAPI MmZeroPageThread(VOID) { PKTHREAD Thread = KeGetCurrentThread(); PVOID StartAddress, EndAddress; PVOID WaitObjects[2]; KIRQL OldIrql; PVOID ZeroAddress; PFN_NUMBER PageIndex, FreePage; PMMPFN Pfn1; /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); /* Set our priority to 0 */ Thread->BasePriority = 0; KeSetPriorityThread(Thread, 0); /* Setup the wait objects */ WaitObjects[0] = &MmZeroingPageEvent; // WaitObjects[1] = &PoSystemIdleTimer; FIXME: Implement idle timer while (TRUE) { KeWaitForMultipleObjects(1, // 2 WaitObjects, WaitAny, WrFreePage, KernelMode, FALSE, NULL, NULL); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); while (TRUE) { if (!MmFreePageListHead.Total) { MmZeroingPageThreadActive = FALSE; KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); break; } PageIndex = MmFreePageListHead.Flink; ASSERT(PageIndex != LIST_HEAD); Pfn1 = MiGetPfnEntry(PageIndex); MI_SET_USAGE(MI_USAGE_ZERO_LOOP); MI_SET_PROCESS2("Kernel 0 Loop"); FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex)); /* The first global free page should also be the first on its own list */ if (FreePage != PageIndex) { KeBugCheckEx(PFN_LIST_CORRUPT, 0x8F, FreePage, PageIndex, 0); } Pfn1->u1.Flink = LIST_HEAD; KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); ZeroAddress = MiMapPagesInZeroSpace(Pfn1, 1); ASSERT(ZeroAddress); RtlZeroMemory(ZeroAddress, PAGE_SIZE); MiUnmapPagesInZeroSpace(ZeroAddress, 1); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MiInsertPageInList(&MmZeroedPageListHead, PageIndex); } } }