NTSTATUS MiCcPutPagesInTransition ( IN PMI_READ_INFO MiReadInfo ) /*++ Routine Description: This routine allocates physical memory for the specified read-list and puts all the pages in transition (so collided faults from other threads for these same pages remain coherent). I/O for any pages not already resident are issued here. The caller must wait for their completion. Arguments: MiReadInfo - Supplies a pointer to the read-list. Return Value: STATUS_SUCCESS - all the pages were already resident, reference counts have been applied and no I/O needs to be waited for. STATUS_ISSUE_PAGING_IO - the I/O has been issued and the caller must wait. Various other failure status values indicate the operation failed. Environment: Kernel mode. PASSIVE_LEVEL. --*/ { NTSTATUS status; PMMPTE LocalPrototypePte; PVOID StartingVa; PFN_NUMBER MdlPages; KIRQL OldIrql; MMPTE PteContents; PFN_NUMBER PageFrameIndex; PFN_NUMBER ResidentAvailableCharge; PPFN_NUMBER IoPage; PPFN_NUMBER ApiPage; PPFN_NUMBER Page; PPFN_NUMBER DestinationPage; ULONG PageColor; PMMPTE PointerPte; PMMPTE *ProtoPteArray; PMMPTE *EndProtoPteArray; PFN_NUMBER DummyPage; PMDL Mdl; PMDL FreeMdl; PMMPFN PfnProto; PMMPFN Pfn1; PMMPFN DummyPfn1; ULONG i; PFN_NUMBER DummyTrim; ULONG NumberOfPagesNeedingIo; MMPTE TempPte; PMMPTE PointerPde; PEPROCESS CurrentProcess; PMMINPAGE_SUPPORT InPageSupport; PKPRCB Prcb; ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL); MiReadInfo->DummyPagePfn = NULL; FreeMdl = NULL; CurrentProcess = PsGetCurrentProcess(); PfnProto = NULL; PointerPde = NULL; InPageSupport = MiReadInfo->InPageSupport; Mdl = MI_EXTRACT_PREFETCH_MDL (InPageSupport); ASSERT (Mdl == MiReadInfo->IoMdl); IoPage = (PPFN_NUMBER)(Mdl + 1); ApiPage = (PPFN_NUMBER)(MiReadInfo->ApiMdl + 1); StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset); MdlPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES (StartingVa, Mdl->ByteCount); if (MdlPages + 1 > MAXUSHORT) { // // The PFN ReferenceCount for the dummy page could wrap, refuse the // request. // return STATUS_INSUFFICIENT_RESOURCES; } NumberOfPagesNeedingIo = 0; ProtoPteArray = (PMMPTE *)InPageSupport->BasePte; EndProtoPteArray = ProtoPteArray + MdlPages; ASSERT (*ProtoPteArray != NULL); LOCK_PFN (OldIrql); // // Ensure sufficient pages exist for the transfer plus the dummy page. // if (((SPFN_NUMBER)MdlPages > (SPFN_NUMBER)(MmAvailablePages - MM_HIGH_LIMIT)) || (MI_NONPAGEABLE_MEMORY_AVAILABLE() <= (SPFN_NUMBER)MdlPages)) { UNLOCK_PFN (OldIrql); return STATUS_INSUFFICIENT_RESOURCES; } // // Charge resident available immediately as the PFN lock may get released // and reacquired below before all the pages have been locked down. // Note the dummy page is immediately charged separately. // MI_DECREMENT_RESIDENT_AVAILABLE (MdlPages, MM_RESAVAIL_ALLOCATE_BUILDMDL); ResidentAvailableCharge = MdlPages; // // Allocate a dummy page to map discarded pages that aren't skipped. // DummyPage = MiRemoveAnyPage (0); Pfn1 = MI_PFN_ELEMENT (DummyPage); ASSERT (Pfn1->u2.ShareCount == 0); ASSERT (Pfn1->u3.e2.ReferenceCount == 0); MiInitializePfnForOtherProcess (DummyPage, MI_PF_DUMMY_PAGE_PTE, 0); // // Give the page a containing frame so MiIdentifyPfn won't crash. // Pfn1->u4.PteFrame = PsInitialSystemProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT; // // Always bias the reference count by 1 and charge for this locked page // up front so the myriad increments and decrements don't get slowed // down with needless checking. // Pfn1->u3.e1.PrototypePte = 0; MI_ADD_LOCKED_PAGE_CHARGE (Pfn1); Pfn1->u3.e1.ReadInProgress = 1; MiReadInfo->DummyPagePfn = Pfn1; DummyPfn1 = Pfn1; DummyPfn1->u3.e2.ReferenceCount = (USHORT)(DummyPfn1->u3.e2.ReferenceCount + MdlPages); // // Properly initialize the inpage support block fields we overloaded. // InPageSupport->BasePte = *ProtoPteArray; // // Build the proper InPageSupport and MDL to describe this run. // for (; ProtoPteArray < EndProtoPteArray; ProtoPteArray += 1, IoPage += 1, ApiPage += 1) { // // Fill the MDL entry for this RLE. // PointerPte = *ProtoPteArray; ASSERT (PointerPte != NULL); // // The PointerPte better be inside a prototype PTE allocation // so that subsequent page trims update the correct PTEs. // ASSERT (((PointerPte >= (PMMPTE)MmPagedPoolStart) && (PointerPte <= (PMMPTE)MmPagedPoolEnd)) || ((PointerPte >= (PMMPTE)MmSpecialPoolStart) && (PointerPte <= (PMMPTE)MmSpecialPoolEnd))); // // Check the state of this prototype PTE now that the PFN lock is held. // If the page is not resident, the PTE must be put in transition with // read in progress before the PFN lock is released. // // // Lock page containing prototype PTEs in memory by // incrementing the reference count for the page. // Unlock any page locked earlier containing prototype PTEs if // the containing page is not the same for both. // if (PfnProto != NULL) { if (PointerPde != MiGetPteAddress (PointerPte)) { ASSERT (PfnProto->u3.e2.ReferenceCount > 1); MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto); PfnProto = NULL; } } if (PfnProto == NULL) { ASSERT (!MI_IS_PHYSICAL_ADDRESS (PointerPte)); PointerPde = MiGetPteAddress (PointerPte); if (PointerPde->u.Hard.Valid == 0) { MiMakeSystemAddressValidPfn (PointerPte, OldIrql); } PfnProto = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber); MI_ADD_LOCKED_PAGE_CHARGE (PfnProto); ASSERT (PfnProto->u3.e2.ReferenceCount > 1); } recheck: PteContents = *PointerPte; // LWFIX: are zero or dzero ptes possible here ? ASSERT (PteContents.u.Long != 0); if (PteContents.u.Hard.Valid == 1) { PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&PteContents); Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->u3.e1.PrototypePte == 1); MI_ADD_LOCKED_PAGE_CHARGE (Pfn1); *ApiPage = PageFrameIndex; *IoPage = DummyPage; continue; } if ((PteContents.u.Soft.Prototype == 0) && (PteContents.u.Soft.Transition == 1)) { // // The page is in transition. If there is an inpage still in // progress, wait for it to complete. Reference the PFN and // then march on. // PageFrameIndex = MI_GET_PAGE_FRAME_FROM_TRANSITION_PTE (&PteContents); Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->u3.e1.PrototypePte == 1); if (Pfn1->u4.InPageError) { // // There was an in-page read error and there are other // threads colliding for this page, delay to let the // other threads complete and then retry. // UNLOCK_PFN (OldIrql); KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond); LOCK_PFN (OldIrql); goto recheck; } if (Pfn1->u3.e1.ReadInProgress) { // LWFIX - start with temp\aw.c } // // PTE refers to a normal transition PTE. // ASSERT ((SPFN_NUMBER)MmAvailablePages >= 0); if (MmAvailablePages == 0) { // // This can only happen if the system is utilizing a hardware // compression cache. This ensures that only a safe amount // of the compressed virtual cache is directly mapped so that // if the hardware gets into trouble, we can bail it out. // UNLOCK_PFN (OldIrql); KeDelayExecutionThread (KernelMode, FALSE, (PLARGE_INTEGER)&MmHalfSecond); LOCK_PFN (OldIrql); goto recheck; } // // The PFN reference count will be 1 already here if the // modified writer has begun a write of this page. Otherwise // it's ordinarily 0. // MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1); *IoPage = DummyPage; *ApiPage = PageFrameIndex; continue; } // LWFIX: need to handle protos that are now pagefile (or dzero) // backed - prefetching it from the file here would cause us to lose // the contents. Note this can happen for session-space images // as we back modified (ie: for relocation fixups or IAT // updated) portions from the pagefile. remove the assert below too. ASSERT (PteContents.u.Soft.Prototype == 1); if ((MmAvailablePages < MM_HIGH_LIMIT) && (MiEnsureAvailablePageOrWait (NULL, OldIrql))) { // // Had to wait so recheck all state. // goto recheck; } NumberOfPagesNeedingIo += 1; // // Allocate a physical page. // PageColor = MI_PAGE_COLOR_VA_PROCESS ( MiGetVirtualAddressMappedByPte (PointerPte), &CurrentProcess->NextPageColor); PageFrameIndex = MiRemoveAnyPage (PageColor); Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->u3.e2.ReferenceCount == 0); ASSERT (Pfn1->u2.ShareCount == 0); ASSERT (PointerPte->u.Hard.Valid == 0); // // Initialize read-in-progress PFN. // MiInitializePfn (PageFrameIndex, PointerPte, 0); // // These pieces of MiInitializePfn initialization are overridden // here as these pages are only going into prototype // transition and not into any page tables. // Pfn1->u3.e1.PrototypePte = 1; Pfn1->u2.ShareCount -= 1; ASSERT (Pfn1->u2.ShareCount == 0); Pfn1->u3.e1.PageLocation = ZeroedPageList; Pfn1->u3.e2.ReferenceCount -= 1; ASSERT (Pfn1->u3.e2.ReferenceCount == 0); MI_ADD_LOCKED_PAGE_CHARGE_FOR_MODIFIED_PAGE (Pfn1); // // Initialize the I/O specific fields. // Pfn1->u1.Event = &InPageSupport->Event; Pfn1->u3.e1.ReadInProgress = 1; ASSERT (Pfn1->u4.InPageError == 0); // // Increment the PFN reference count in the control area for // the subsection. // MiReadInfo->ControlArea->NumberOfPfnReferences += 1; // // Put the prototype PTE into the transition state. // MI_MAKE_TRANSITION_PTE (TempPte, PageFrameIndex, PointerPte->u.Soft.Protection, PointerPte); MI_WRITE_INVALID_PTE (PointerPte, TempPte); *IoPage = PageFrameIndex; *ApiPage = PageFrameIndex; } // // If all the pages were resident, dereference the dummy page references // now and notify our caller that I/O is not necessary. // if (NumberOfPagesNeedingIo == 0) { ASSERT (DummyPfn1->u3.e2.ReferenceCount > MdlPages); DummyPfn1->u3.e2.ReferenceCount = (USHORT)(DummyPfn1->u3.e2.ReferenceCount - MdlPages); // // Unlock page containing prototype PTEs. // if (PfnProto != NULL) { ASSERT (PfnProto->u3.e2.ReferenceCount > 1); MI_REMOVE_LOCKED_PAGE_CHARGE_AND_DECREF (PfnProto); } UNLOCK_PFN (OldIrql); // // Return the upfront resident available charge as the // individual charges have all been made at this point. // MI_INCREMENT_RESIDENT_AVAILABLE (ResidentAvailableCharge, MM_RESAVAIL_FREE_BUILDMDL_EXCESS); return STATUS_SUCCESS; } // // Carefully trim leading dummy pages. // Page = (PPFN_NUMBER)(Mdl + 1); DummyTrim = 0; for (i = 0; i < MdlPages - 1; i += 1) { if (*Page == DummyPage) { DummyTrim += 1; Page += 1; } else { break; } } if (DummyTrim != 0) { Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER))); Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE); ASSERT (Mdl->ByteCount != 0); InPageSupport->ReadOffset.QuadPart += (DummyTrim * PAGE_SIZE); DummyPfn1->u3.e2.ReferenceCount = (USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim); // // Shuffle down the PFNs in the MDL. // Recalculate BasePte to adjust for the shuffle. // Pfn1 = MI_PFN_ELEMENT (*Page); ASSERT (Pfn1->PteAddress->u.Hard.Valid == 0); ASSERT ((Pfn1->PteAddress->u.Soft.Prototype == 0) && (Pfn1->PteAddress->u.Soft.Transition == 1)); InPageSupport->BasePte = Pfn1->PteAddress; DestinationPage = (PPFN_NUMBER)(Mdl + 1); do { *DestinationPage = *Page; DestinationPage += 1; Page += 1; i += 1; } while (i < MdlPages); MdlPages -= DummyTrim; } // // Carefully trim trailing dummy pages. // ASSERT (MdlPages != 0); Page = (PPFN_NUMBER)(Mdl + 1) + MdlPages - 1; if (*Page == DummyPage) { ASSERT (MdlPages >= 2); // // Trim the last page specially as it may be a partial page. // Mdl->Size -= sizeof(PFN_NUMBER); if (BYTE_OFFSET(Mdl->ByteCount) != 0) { Mdl->ByteCount &= ~(PAGE_SIZE - 1); } else { Mdl->ByteCount -= PAGE_SIZE; } ASSERT (Mdl->ByteCount != 0); DummyPfn1->u3.e2.ReferenceCount -= 1; // // Now trim any other trailing pages. // Page -= 1; DummyTrim = 0; while (Page != ((PPFN_NUMBER)(Mdl + 1))) { if (*Page != DummyPage) { break; } DummyTrim += 1; Page -= 1; } if (DummyTrim != 0) { ASSERT (Mdl->Size > (USHORT)(DummyTrim * sizeof(PFN_NUMBER))); Mdl->Size = (USHORT)(Mdl->Size - (DummyTrim * sizeof(PFN_NUMBER))); Mdl->ByteCount -= (ULONG)(DummyTrim * PAGE_SIZE); DummyPfn1->u3.e2.ReferenceCount = (USHORT)(DummyPfn1->u3.e2.ReferenceCount - DummyTrim); } ASSERT (MdlPages > DummyTrim + 1); MdlPages -= (DummyTrim + 1); #if DBG StartingVa = (PVOID)((PCHAR)Mdl->StartVa + Mdl->ByteOffset); ASSERT (MdlPages == ADDRESS_AND_SIZE_TO_SPAN_PAGES(StartingVa, Mdl->ByteCount)); #endif } // // If the MDL is not already embedded in the inpage block, see if its // final size qualifies it - if so, embed it now. // if ((Mdl != &InPageSupport->Mdl) && (Mdl->ByteCount <= (MM_MAXIMUM_READ_CLUSTER_SIZE + 1) * PAGE_SIZE)){ #if DBG RtlFillMemoryUlong (&InPageSupport->Page[0], (MM_MAXIMUM_READ_CLUSTER_SIZE+1) * sizeof (PFN_NUMBER), 0xf1f1f1f1); #endif RtlCopyMemory (&InPageSupport->Mdl, Mdl, Mdl->Size); FreeMdl = Mdl; Mdl = &InPageSupport->Mdl; ASSERT (((ULONG_PTR)Mdl & (sizeof(QUAD) - 1)) == 0); InPageSupport->u1.e1.PrefetchMdlHighBits = ((ULONG_PTR)Mdl >> 3); }
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes) { PFN_NUMBER PageFrameNumber; PFN_COUNT SizeInPages, PageTableCount; ULONG i; KIRQL OldIrql; PLIST_ENTRY NextEntry, NextHead, LastHead; PMMPTE PointerPte, StartPte; PMMPDE PointerPde; ULONG EndAllocation; MMPTE TempPte; MMPDE TempPde; PMMPFN Pfn1; PVOID BaseVa, BaseVaStart; PMMFREE_POOL_ENTRY FreeEntry; PKSPIN_LOCK_QUEUE LockQueue; // // Figure out how big the allocation is in pages // SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes); // // Check for overflow // if (SizeInPages == 0) { // // Fail // return NULL; } // // Handle paged pool // if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Lock the paged pool mutex // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Find some empty allocation space // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, MmPagedPoolInfo.PagedPoolHint); if (i == 0xFFFFFFFF) { // // Get the page bit count // i = ((SizeInPages - 1) / PTE_COUNT) + 1; DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages); // // Check if there is enougn paged pool expansion space left // if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } // // Check if we'll have to expand past the last PTE we have available // if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // We can only support this much then // PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool); PageTableCount = (PFN_COUNT)(PointerPde + 1 - MmPagedPoolInfo.NextPdeForPagedPoolExpansion); ASSERT(PageTableCount < i); i = PageTableCount; } else { // // Otherwise, there is plenty of space left for this expansion // PageTableCount = i; } // // Get the template PDE we'll use to expand // TempPde = ValidKernelPde; // // Get the first PTE in expansion space // PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion; BaseVa = MiPdeToPte(PointerPde); BaseVaStart = BaseVa; // // Lock the PFN database and loop pages // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // It should not already be valid // ASSERT(PointerPde->u.Hard.Valid == 0); /* Request a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPde.u.Hard.PageFrameNumber = PageFrameNumber; #if (_MI_PAGING_LEVELS >= 3) /* On PAE/x64 systems, there's no double-buffering */ ASSERT(FALSE); #else // // Save it into our double-buffered system page directory // MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde; /* Initialize the PFN */ MiInitializePfnForOtherProcess(PageFrameNumber, (PMMPTE)PointerPde, MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]); /* Write the actual PDE now */ // MI_WRITE_VALID_PDE(PointerPde, TempPde); #endif // // Move on to the next expansion address // PointerPde++; BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE); i--; } while (i > 0); // // Release the PFN database lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // These pages are now available, clear their availablity bits // EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion - (PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) * PTE_COUNT; RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, EndAllocation, PageTableCount * PTE_COUNT); // // Update the next expansion location // MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount; // // Zero out the newly available memory // RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE); // // Now try consuming the pages again // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, 0); if (i == 0xFFFFFFFF) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } } // // Update the pool hint if the request was just one page // if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1; // // Update the end bitmap so we know the bounds of this allocation when // the time comes to free it // EndAllocation = i + SizeInPages - 1; RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation); // // Now we can release the lock (it mainly protects the bitmap) // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // Now figure out where this allocation starts // BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT)); // // Flush the TLB // KeFlushEntireTb(TRUE, TRUE); /* Setup a demand-zero writable PTE */ MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE); // // Find the first and last PTE, then loop them all // PointerPte = MiAddressToPte(BaseVa); StartPte = PointerPte + SizeInPages; do { // // Write the demand zero PTE and keep going // MI_WRITE_INVALID_PTE(PointerPte, TempPte); } while (++PointerPte < StartPte); // // Return the allocation address to the caller // return BaseVa; } // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Allocations of less than 4 pages go into their individual buckets // i = SizeInPages - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; // // Loop through all the free page lists based on the page index // NextHead = &MmNonPagedPoolFreeListHead[i]; LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS]; // // Acquire the nonpaged pool lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); do { // // Now loop through all the free page entries in this given list // NextEntry = NextHead->Flink; while (NextEntry != NextHead) { /* Is freed non paged pool enabled */ if (MmProtectFreedNonPagedPool) { /* We need to be able to touch this page, unprotect it */ MiUnProtectFreeNonPagedPool(NextEntry, 0); } // // Grab the entry and see if it can handle our allocation // FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List); ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE); if (FreeEntry->Size >= SizeInPages) { // // It does, so consume the pages from here // FreeEntry->Size -= SizeInPages; // // The allocation will begin in this free page area // BaseVa = (PVOID)((ULONG_PTR)FreeEntry + (FreeEntry->Size << PAGE_SHIFT)); /* Remove the item from the list, depending if pool is protected */ if (MmProtectFreedNonPagedPool) MiProtectedPoolRemoveEntryList(&FreeEntry->List); else RemoveEntryList(&FreeEntry->List); // // However, check if its' still got space left // if (FreeEntry->Size != 0) { /* Check which list to insert this entry into */ i = FreeEntry->Size - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; /* Insert the entry into the free list head, check for prot. pool */ if (MmProtectFreedNonPagedPool) MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE); else InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List); /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } // // Grab the PTE for this allocation // PointerPte = MiAddressToPte(BaseVa); ASSERT(PointerPte->u.Hard.Valid == 1); // // Grab the PFN NextEntry and index // Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); // // Now mark it as the beginning of an allocation // ASSERT(Pfn1->u3.e1.StartOfAllocation == 0); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as special pool if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) { Pfn1->u4.VerifierAllocation = 1; } // // Check if the allocation is larger than one page // if (SizeInPages != 1) { // // Navigate to the last PFN entry and PTE // PointerPte += SizeInPages - 1; ASSERT(PointerPte->u.Hard.Valid == 1); Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber); } // // Mark this PFN as the last (might be the same as the first) // ASSERT(Pfn1->u3.e1.EndOfAllocation == 0); Pfn1->u3.e1.EndOfAllocation = 1; // // Release the nonpaged pool lock, and return the allocation // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); return BaseVa; } // // Try the next free page entry // NextEntry = FreeEntry->List.Flink; /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } } while (++NextHead < LastHead); // // If we got here, we're out of space. // Start by releasing the lock // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Allocate some system PTEs // StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion); PointerPte = StartPte; if (StartPte == NULL) { // // Ran out of memory // DPRINT1("Out of NP Expansion Pool\n"); return NULL; } // // Acquire the pool lock now // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); // // Lock the PFN database too // LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock]; KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); // // Loop the pages // TempPte = ValidKernelPte; do { /* Allocate a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->PteAddress = PointerPte; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u4.VerifierAllocation = 0; /* Write the PTE for it */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MI_WRITE_VALID_PTE(PointerPte++, TempPte); } while (--SizeInPages > 0); // // This is the last page // Pfn1->u3.e1.EndOfAllocation = 1; // // Get the first page and mark it as such // Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as a verifier allocation if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1; // // Release the PFN and nonpaged pool lock // KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Return the address // return MiPteToAddress(StartPte); }
VOID MmZeroPageThread ( VOID ) /*++ Routine Description: Implements the NT zeroing page thread. This thread runs at priority zero and removes a page from the free list, zeroes it, and places it on the zeroed page list. Arguments: StartContext - not used. Return Value: None. Environment: Kernel mode. --*/ { PVOID EndVa; KIRQL OldIrql; ULONG PageFrame; PMMPFN Pfn1; PVOID StartVa; PKTHREAD Thread; PVOID ZeroBase; ULONG i; // // Before this becomes the zero page thread, free the kernel // initialization code. // MiFindInitializationCode (&StartVa, &EndVa); if (StartVa != NULL) { MiFreeInitializationCode (StartVa, EndVa); } // // The following code sets the current thread's base priority to zero // and then sets its current priority to zero. This ensures that the // thread always runs at a priority of zero. // Thread = KeGetCurrentThread(); Thread->BasePriority = 0; KeSetPriorityThread (Thread, 0); // // Loop forever zeroing pages. // do { // // Wait until there are at least MmZeroPageMinimum pages // on the free list. // KeWaitForSingleObject (&MmZeroingPageEvent, WrFreePage, KernelMode, FALSE, (PLARGE_INTEGER)NULL); LOCK_PFN_WITH_TRY (OldIrql); do { if ((volatile)MmFreePageListHead.Total == 0) { // // No pages on the free list at this time, wait for // some more. // MmZeroingPageThreadActive = FALSE; UNLOCK_PFN (OldIrql); break; } else { #if MM_MAXIMUM_NUMBER_OF_COLORS > 1 for (i = 0; i < MM_MAXIMUM_NUMBER_OF_COLORS; i++) { PageFrame = MmFreePagesByPrimaryColor[FreePageList][i].Flink; if (PageFrame != MM_EMPTY_LIST) { break; } } #else //MM_MAXIMUM_NUMBER_OF_COLORS > 1 PageFrame = MmFreePageListHead.Flink; #endif //MM_MAXIMUM_NUMBER_OF_COLORS > 1 Pfn1 = MI_PFN_ELEMENT(PageFrame); ASSERT (PageFrame != MM_EMPTY_LIST); Pfn1 = MI_PFN_ELEMENT(PageFrame); MiRemoveAnyPage (MI_GET_SECONDARY_COLOR (PageFrame, Pfn1)); // // Zero the page using the last color used to map the page. // #if defined(_X86_) ZeroBase = MiMapPageToZeroInHyperSpace (PageFrame); UNLOCK_PFN (OldIrql); RtlZeroMemory (ZeroBase, PAGE_SIZE); #elif defined(_PPC_) UNLOCK_PFN (OldIrql); KeZeroPage(PageFrame); #else ZeroBase = (PVOID)(Pfn1->u3.e1.PageColor << PAGE_SHIFT); UNLOCK_PFN (OldIrql); HalZeroPage(ZeroBase, ZeroBase, PageFrame); #endif //X86 LOCK_PFN_WITH_TRY (OldIrql); MiInsertPageInList (MmPageLocationList[ZeroedPageList], PageFrame); } } while(TRUE); } while (TRUE); }
BOOLEAN NTAPI MmCreateProcessAddressSpace(IN ULONG MinWs, IN PEPROCESS Process, OUT PULONG_PTR DirectoryTableBase) { KIRQL OldIrql; PFN_NUMBER TableBasePfn, HyperPfn, HyperPdPfn, HyperPtPfn, WorkingSetPfn; PMMPTE SystemPte; MMPTE TempPte, PdePte; ULONG TableIndex; PMMPTE PageTablePointer; /* Make sure we don't already have a page directory setup */ ASSERT(Process->Pcb.DirectoryTableBase[0] == 0); ASSERT(Process->Pcb.DirectoryTableBase[1] == 0); ASSERT(Process->WorkingSetPage == 0); /* Choose a process color */ Process->NextPageColor = (USHORT)RtlRandom(&MmProcessColorSeed); /* Setup the hyperspace lock */ KeInitializeSpinLock(&Process->HyperSpaceLock); /* Lock PFN database */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Get a page for the table base and one for hyper space. The PFNs for these pages will be initialized in MmInitializeProcessAddressSpace, when we are already attached to the process. */ TableBasePfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); HyperPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); HyperPdPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); HyperPtPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); WorkingSetPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); /* Release PFN lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); /* Zero pages */ /// FIXME: MiZeroPhysicalPage(HyperPfn); MiZeroPhysicalPage(WorkingSetPfn); /* Set the base directory pointers */ Process->WorkingSetPage = WorkingSetPfn; DirectoryTableBase[0] = TableBasePfn << PAGE_SHIFT; DirectoryTableBase[1] = HyperPfn << PAGE_SHIFT; /* Get a PTE to map the page directory */ SystemPte = MiReserveSystemPtes(1, SystemPteSpace); ASSERT(SystemPte != NULL); /* Get its address */ PageTablePointer = MiPteToAddress(SystemPte); /* Build the PTE for the page directory and map it */ PdePte = ValidKernelPte; PdePte.u.Hard.PageFrameNumber = TableBasePfn; *SystemPte = PdePte; /// architecture specific //MiInitializePageDirectoryForProcess( /* Copy the kernel mappings and zero out the rest */ TableIndex = PXE_PER_PAGE / 2; RtlZeroMemory(PageTablePointer, TableIndex * sizeof(MMPTE)); RtlCopyMemory(PageTablePointer + TableIndex, MiAddressToPxe(0) + TableIndex, PAGE_SIZE - TableIndex * sizeof(MMPTE)); /* Sanity check */ ASSERT(MiAddressToPxi(MmHyperSpaceEnd) >= TableIndex); /* Setup a PTE for the page directory mappings */ TempPte = ValidKernelPte; /* Update the self mapping of the PML4 */ TableIndex = MiAddressToPxi((PVOID)PXE_SELFMAP); TempPte.u.Hard.PageFrameNumber = TableBasePfn; PageTablePointer[TableIndex] = TempPte; /* Write the PML4 entry for hyperspace */ TableIndex = MiAddressToPxi((PVOID)HYPER_SPACE); TempPte.u.Hard.PageFrameNumber = HyperPfn; PageTablePointer[TableIndex] = TempPte; /* Map the hyperspace PDPT to the system PTE */ PdePte.u.Hard.PageFrameNumber = HyperPfn; *SystemPte = PdePte; __invlpg(PageTablePointer); /* Write the hyperspace entry for the first PD */ TempPte.u.Hard.PageFrameNumber = HyperPdPfn; PageTablePointer[0] = TempPte; /* Map the hyperspace PD to the system PTE */ PdePte.u.Hard.PageFrameNumber = HyperPdPfn; *SystemPte = PdePte; __invlpg(PageTablePointer); /* Write the hyperspace entry for the first PT */ TempPte.u.Hard.PageFrameNumber = HyperPtPfn; PageTablePointer[0] = TempPte; /* Map the hyperspace PT to the system PTE */ PdePte.u.Hard.PageFrameNumber = HyperPtPfn; *SystemPte = PdePte; __invlpg(PageTablePointer); /* Write the hyperspace PTE for the working set list index */ TempPte.u.Hard.PageFrameNumber = WorkingSetPfn; TableIndex = MiAddressToPti(MmWorkingSetList); PageTablePointer[TableIndex] = TempPte; /// end architecture specific /* Release the system PTE */ MiReleaseSystemPtes(SystemPte, 1, SystemPteSpace); /* Switch to phase 1 initialization */ ASSERT(Process->AddressSpaceInitialized == 0); Process->AddressSpaceInitialized = 1; return TRUE; }
VOID NTAPI MmZeroPageThread(VOID) { PKTHREAD Thread = KeGetCurrentThread(); PVOID StartAddress, EndAddress; PVOID WaitObjects[2]; KIRQL OldIrql; PVOID ZeroAddress; PFN_NUMBER PageIndex, FreePage; PMMPFN Pfn1; /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); /* Set our priority to 0 */ Thread->BasePriority = 0; KeSetPriorityThread(Thread, 0); /* Setup the wait objects */ WaitObjects[0] = &MmZeroingPageEvent; // WaitObjects[1] = &PoSystemIdleTimer; FIXME: Implement idle timer while (TRUE) { KeWaitForMultipleObjects(1, // 2 WaitObjects, WaitAny, WrFreePage, KernelMode, FALSE, NULL, NULL); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); while (TRUE) { if (!MmFreePageListHead.Total) { MmZeroingPageThreadActive = FALSE; KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); break; } PageIndex = MmFreePageListHead.Flink; ASSERT(PageIndex != LIST_HEAD); Pfn1 = MiGetPfnEntry(PageIndex); MI_SET_USAGE(MI_USAGE_ZERO_LOOP); MI_SET_PROCESS2("Kernel 0 Loop"); FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex)); /* The first global free page should also be the first on its own list */ if (FreePage != PageIndex) { KeBugCheckEx(PFN_LIST_CORRUPT, 0x8F, FreePage, PageIndex, 0); } Pfn1->u1.Flink = LIST_HEAD; KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); ZeroAddress = MiMapPagesInZeroSpace(Pfn1, 1); ASSERT(ZeroAddress); RtlZeroMemory(ZeroAddress, PAGE_SIZE); MiUnmapPagesInZeroSpace(ZeroAddress, 1); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MiInsertPageInList(&MmZeroedPageListHead, PageIndex); } } }
PVOID NTAPI MmAllocateSpecialPool(SIZE_T NumberOfBytes, ULONG Tag, POOL_TYPE PoolType, ULONG SpecialType) { KIRQL Irql; MMPTE TempPte = ValidKernelPte; PMMPTE PointerPte; PFN_NUMBER PageFrameNumber; LARGE_INTEGER TickCount; PVOID Entry; PPOOL_HEADER Header; DPRINT1("MmAllocateSpecialPool(%x %x %x %x)\n", NumberOfBytes, Tag, PoolType, SpecialType); /* Check if the pool is initialized and quit if it's not */ if (!MiSpecialPoolFirstPte) return NULL; /* Get the pool type */ PoolType &= BASE_POOL_TYPE_MASK; /* Check whether current IRQL matches the pool type */ Irql = KeGetCurrentIrql(); if (((PoolType == PagedPool) && (Irql > APC_LEVEL)) || ((PoolType != PagedPool) && (Irql > DISPATCH_LEVEL))) { /* Bad caller */ KeBugCheckEx(SPECIAL_POOL_DETECTED_MEMORY_CORRUPTION, Irql, PoolType, NumberOfBytes, 0x30); } /* TODO: Take into account various limitations */ /*if ((PoolType != NonPagedPool) && MiSpecialPagesNonPaged > MiSpecialPagesNonPagedMaximum)*/ /* Lock PFN database */ Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Reject allocation in case amount of available pages is too small */ if (MmAvailablePages < 0x100) { /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); DPRINT1("Special pool: MmAvailablePages 0x%x is too small\n", MmAvailablePages); return NULL; } /* Reject allocation if special pool PTE list is exhausted */ if (MiSpecialPoolFirstPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) { /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); DPRINT1("Special pool: No PTEs left!\n"); /* TODO: Expand the special pool */ return NULL; } /* Save allocation time */ KeQueryTickCount(&TickCount); /* Get a pointer to the first PTE */ PointerPte = MiSpecialPoolFirstPte; /* Set the first PTE pointer to the next one in the list */ MiSpecialPoolFirstPte = MmSystemPteBase + PointerPte->u.List.NextEntry; /* Allocate a physical page */ PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Initialize PFN and make it valid */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MiInitializePfnAndMakePteValid(PageFrameNumber, PointerPte, TempPte); /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); /* Put some content into the page. Low value of tick count would do */ Entry = MiPteToAddress(PointerPte); RtlFillMemory(Entry, PAGE_SIZE, TickCount.LowPart); /* Calculate header and entry addresses */ if ((SpecialType != 0) && ((SpecialType == 1) || (!MmSpecialPoolCatchOverruns))) { /* We catch underruns. Data is at the beginning of the page */ Header = (PPOOL_HEADER)((PUCHAR)Entry + PAGE_SIZE - sizeof(POOL_HEADER)); } else { /* We catch overruns. Data is at the end of the page */ Header = (PPOOL_HEADER)Entry; Entry = (PVOID)((ULONG_PTR)((PUCHAR)Entry - NumberOfBytes + PAGE_SIZE) & ~((LONG_PTR)sizeof(POOL_HEADER) - 1)); } /* Initialize the header */ RtlZeroMemory(Header, sizeof(POOL_HEADER)); /* Save allocation size there */ Header->Ulong1 = (ULONG)NumberOfBytes; /* Make sure it's all good */ ASSERT((NumberOfBytes <= PAGE_SIZE - sizeof(POOL_HEADER)) && (PAGE_SIZE <= 32 * 1024)); /* Mark it as paged or nonpaged */ if (PoolType == PagedPool) { /* Add pagedpool flag into the pool header too */ Header->Ulong1 |= SPECIAL_POOL_PAGED; /* Also mark the next PTE as special-pool-paged */ PointerPte[1].u.Soft.PageFileHigh |= SPECIAL_POOL_PAGED_PTE; } else { /* Mark the next PTE as special-pool-nonpaged */ PointerPte[1].u.Soft.PageFileHigh |= SPECIAL_POOL_NONPAGED_PTE; } /* Finally save tag and put allocation time into the header's blocksize. That time will be used to check memory consistency within the allocated page. */ Header->PoolTag = Tag; Header->BlockSize = (USHORT)TickCount.LowPart; DPRINT1("%p\n", Entry); return Entry; }
LOGICAL FASTCALL MiCopyOnWrite ( IN PVOID FaultingAddress, IN PMMPTE PointerPte ) /*++ Routine Description: This routine performs a copy on write operation for the specified virtual address. Arguments: FaultingAddress - Supplies the virtual address which caused the fault. PointerPte - Supplies the pointer to the PTE which caused the page fault. Return Value: Returns TRUE if the page was actually split, FALSE if not. Environment: Kernel mode, APCs disabled, working set mutex held. --*/ { MMPTE TempPte; MMPTE TempPte2; PMMPTE MappingPte; PFN_NUMBER PageFrameIndex; PFN_NUMBER NewPageIndex; PVOID CopyTo; PVOID CopyFrom; KIRQL OldIrql; PMMPFN Pfn1; PEPROCESS CurrentProcess; PMMCLONE_BLOCK CloneBlock; PMMCLONE_DESCRIPTOR CloneDescriptor; WSLE_NUMBER WorkingSetIndex; LOGICAL FakeCopyOnWrite; PMMWSL WorkingSetList; PVOID SessionSpace; PLIST_ENTRY NextEntry; PIMAGE_ENTRY_IN_SESSION Image; // // This is called from MmAccessFault, the PointerPte is valid // and the working set mutex ensures it cannot change state. // // Capture the PTE contents to TempPte. // TempPte = *PointerPte; ASSERT (TempPte.u.Hard.Valid == 1); PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte); Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); // // Check to see if this is a prototype PTE with copy on write enabled. // FakeCopyOnWrite = FALSE; CurrentProcess = PsGetCurrentProcess (); CloneBlock = NULL; if (FaultingAddress >= (PVOID) MmSessionBase) { WorkingSetList = MmSessionSpace->Vm.VmWorkingSetList; ASSERT (Pfn1->u3.e1.PrototypePte == 1); SessionSpace = (PVOID) MmSessionSpace; MM_SESSION_SPACE_WS_LOCK_ASSERT (); if (MmSessionSpace->ImageLoadingCount != 0) { NextEntry = MmSessionSpace->ImageList.Flink; while (NextEntry != &MmSessionSpace->ImageList) { Image = CONTAINING_RECORD (NextEntry, IMAGE_ENTRY_IN_SESSION, Link); if ((FaultingAddress >= Image->Address) && (FaultingAddress <= Image->LastAddress)) { if (Image->ImageLoading) { ASSERT (Pfn1->u3.e1.PrototypePte == 1); TempPte.u.Hard.CopyOnWrite = 0; TempPte.u.Hard.Write = 1; // // The page is no longer copy on write, update the PTE // setting both the dirty bit and the accessed bit. // // Even though the page's current backing is the image // file, the modified writer will convert it to // pagefile backing when it notices the change later. // MI_SET_PTE_DIRTY (TempPte); MI_SET_ACCESSED_IN_PTE (&TempPte, 1); MI_WRITE_VALID_PTE_NEW_PROTECTION (PointerPte, TempPte); // // The TB entry must be flushed as the valid PTE with // the dirty bit clear has been fetched into the TB. If // it isn't flushed, another fault is generated as the // dirty bit is not set in the cached TB entry. // MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE); return FALSE; } break; } NextEntry = NextEntry->Flink; } } } else { WorkingSetList = MmWorkingSetList; SessionSpace = NULL; // // If a fork operation is in progress, block until the fork is // completed, then retry the whole operation as the state of // everything may have changed between when the mutexes were // released and reacquired. // if (CurrentProcess->ForkInProgress != NULL) { if (MiWaitForForkToComplete (CurrentProcess) == TRUE) { return FALSE; } } if (TempPte.u.Hard.CopyOnWrite == 0) { // // This is a fork page which is being made private in order // to change the protection of the page. // Do not make the page writable. // FakeCopyOnWrite = TRUE; } } WorkingSetIndex = MiLocateWsle (FaultingAddress, WorkingSetList, Pfn1->u1.WsIndex, FALSE); // // The page must be copied into a new page. // LOCK_PFN (OldIrql); if ((MmAvailablePages < MM_HIGH_LIMIT) && (MiEnsureAvailablePageOrWait (SessionSpace != NULL ? HYDRA_PROCESS : CurrentProcess, OldIrql))) { // // A wait operation was performed to obtain an available // page and the working set mutex and PFN lock have // been released and various things may have changed for // the worse. Rather than examine all the conditions again, // return and if things are still proper, the fault will // be taken again. // UNLOCK_PFN (OldIrql); return FALSE; } // // This must be a prototype PTE. Perform the copy on write. // ASSERT (Pfn1->u3.e1.PrototypePte == 1); // // A page is being copied and made private, the global state of // the shared page needs to be updated at this point on certain // hardware. This is done by ORing the dirty bit into the modify bit in // the PFN element. // // Note that a session page cannot be dirty (no POSIX-style forking is // supported for these drivers). // if (SessionSpace != NULL) { ASSERT ((TempPte.u.Hard.Valid == 1) && (TempPte.u.Hard.Write == 0)); ASSERT (!MI_IS_PTE_DIRTY (TempPte)); NewPageIndex = MiRemoveAnyPage (MI_GET_PAGE_COLOR_FROM_SESSION(MmSessionSpace)); } else { MI_CAPTURE_DIRTY_BIT_TO_PFN (PointerPte, Pfn1); CloneBlock = (PMMCLONE_BLOCK) Pfn1->PteAddress; // // Get a new page with the same color as this page. // NewPageIndex = MiRemoveAnyPage ( MI_PAGE_COLOR_PTE_PROCESS(PageFrameIndex, &CurrentProcess->NextPageColor)); } MiInitializeCopyOnWritePfn (NewPageIndex, PointerPte, WorkingSetIndex, WorkingSetList); UNLOCK_PFN (OldIrql); InterlockedIncrement (&KeGetCurrentPrcb ()->MmCopyOnWriteCount); CopyFrom = PAGE_ALIGN (FaultingAddress); MappingPte = MiReserveSystemPtes (1, SystemPteSpace); if (MappingPte != NULL) { MI_MAKE_VALID_KERNEL_PTE (TempPte2, NewPageIndex, MM_READWRITE, MappingPte); MI_SET_PTE_DIRTY (TempPte2); if (Pfn1->u3.e1.CacheAttribute == MiNonCached) { MI_DISABLE_CACHING (TempPte2); } else if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined) { MI_SET_PTE_WRITE_COMBINE (TempPte2); } MI_WRITE_VALID_PTE (MappingPte, TempPte2); CopyTo = MiGetVirtualAddressMappedByPte (MappingPte); } else { CopyTo = MiMapPageInHyperSpace (CurrentProcess, NewPageIndex, &OldIrql); } KeCopyPage (CopyTo, CopyFrom); if (MappingPte != NULL) { MiReleaseSystemPtes (MappingPte, 1, SystemPteSpace); } else { MiUnmapPageInHyperSpace (CurrentProcess, CopyTo, OldIrql); } if (!FakeCopyOnWrite) { // // If the page was really a copy on write page, make it // accessed, dirty and writable. Also, clear the copy-on-write // bit in the PTE. // MI_SET_PTE_DIRTY (TempPte); TempPte.u.Hard.Write = 1; MI_SET_ACCESSED_IN_PTE (&TempPte, 1); TempPte.u.Hard.CopyOnWrite = 0; } // // Regardless of whether the page was really a copy on write, // the frame field of the PTE must be updated. // TempPte.u.Hard.PageFrameNumber = NewPageIndex; // // If the modify bit is set in the PFN database for the // page, the data cache must be flushed. This is due to the // fact that this process may have been cloned and the cache // still contains stale data destined for the page we are // going to remove. // ASSERT (TempPte.u.Hard.Valid == 1); MI_WRITE_VALID_PTE_NEW_PAGE (PointerPte, TempPte); // // Flush the TB entry for this page. // if (SessionSpace == NULL) { MI_FLUSH_SINGLE_TB (FaultingAddress, FALSE); // // Increment the number of private pages. // CurrentProcess->NumberOfPrivatePages += 1; } else { MI_FLUSH_SINGLE_TB (FaultingAddress, TRUE); ASSERT (Pfn1->u3.e1.PrototypePte == 1); } // // Decrement the share count for the page which was copied // as this PTE no longer refers to it. // LOCK_PFN (OldIrql); MiDecrementShareCount (Pfn1, PageFrameIndex); if (SessionSpace == NULL) { CloneDescriptor = MiLocateCloneAddress (CurrentProcess, (PVOID)CloneBlock); if (CloneDescriptor != NULL) { // // Decrement the reference count for the clone block, // note that this could release and reacquire the mutexes. // MiDecrementCloneBlockReference (CloneDescriptor, CloneBlock, CurrentProcess, NULL, OldIrql); } } UNLOCK_PFN (OldIrql); return TRUE; }