/* * @halfplemented */ VOID NTAPI IopDecrementDeviceObjectRef(IN PDEVICE_OBJECT DeviceObject, IN BOOLEAN UnloadIfUnused) { KIRQL OldIrql; /* Acquire lock */ OldIrql = KeAcquireQueuedSpinLock(LockQueueIoDatabaseLock); ASSERT(DeviceObject->ReferenceCount > 0); if (--DeviceObject->ReferenceCount > 0) { KeReleaseQueuedSpinLock(LockQueueIoDatabaseLock, OldIrql); return; } /* Here, DO is not referenced any longer, check if we have to unload it */ if (UnloadIfUnused || IoGetDevObjExtension(DeviceObject)->ExtensionFlags & (DOE_UNLOAD_PENDING | DOE_DELETE_PENDING | DOE_REMOVE_PENDING)) { /* Unload the driver */ IopUnloadDevice(DeviceObject); } /* Release lock */ KeReleaseQueuedSpinLock(LockQueueIoDatabaseLock, OldIrql); }
NTSTATUS NTAPI MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) { PMM_ALLOCATION_REQUEST Request; PLIST_ENTRY Entry; KIRQL OldIrql; if (Page == 0) { DPRINT1("Tried to release page zero.\n"); KeBugCheck(MEMORY_MANAGEMENT); } KeAcquireSpinLock(&AllocationListLock, &OldIrql); if (MmGetReferenceCountPage(Page) == 1) { (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages) { KeReleaseSpinLock(&AllocationListLock, OldIrql); if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmDereferencePage(Page); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { Entry = RemoveHeadList(&AllocationListHead); Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry); KeReleaseSpinLock(&AllocationListLock, OldIrql); if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); MiZeroPhysicalPage(Page); Request->Page = Page; KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE); } } else { KeReleaseSpinLock(&AllocationListLock, OldIrql); if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmDereferencePage(Page); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } return(STATUS_SUCCESS); }
/* * @implemented */ VOID NTAPI IoReleaseVpbSpinLock(IN KIRQL Irql) { /* Just release the lock */ KeReleaseQueuedSpinLock(LockQueueIoVpbLock, Irql); }
static VOID MmDeletePageTablePfn(PFN_NUMBER PageFrameNumber, ULONG Level) { PMMPTE PageTable; KIRQL OldIrql; PMMPFN PfnEntry; ULONG i, NumberEntries; /* Check if this is a page table */ if (Level > 0) { NumberEntries = (Level == 4) ? MiAddressToPxi(MmHighestUserAddress)+1 : 512; /* Map the page table in hyperspace */ PageTable = (PMMPTE)MmCreateHyperspaceMapping(PageFrameNumber); /* Loop all page table entries */ for (i = 0; i < NumberEntries; i++) { /* Check if the entry is valid */ if (PageTable[i].u.Hard.Valid) { /* Recursively free the page that backs it */ MmDeletePageTablePfn(PageTable[i].u.Hard.PageFrameNumber, Level - 1); } } /* Delete the hyperspace mapping */ MmDeleteHyperspaceMapping(PageTable); } /* Check if this is a legacy allocation */ PfnEntry = MiGetPfnEntry(PageFrameNumber); if (MI_IS_ROS_PFN(PfnEntry)) { /* Free it using the legacy API */ MmReleasePageMemoryConsumer(MC_SYSTEM, PageFrameNumber); } else { OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Free it using the ARM3 API */ MI_SET_PFN_DELETED(PfnEntry); MiDecrementShareCount(PfnEntry, PageFrameNumber); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } }
/* * @implemented */ ULONG FASTCALL IopInterlockedDecrementUlong(IN KSPIN_LOCK_QUEUE_NUMBER Queue, IN PULONG Ulong) { KIRQL Irql; ULONG OldValue; Irql = KeAcquireQueuedSpinLock(Queue); OldValue = (*Ulong)--; KeReleaseQueuedSpinLock(Queue, Irql); return OldValue; }
VOID CcPostWorkQueue( IN PWORK_QUEUE_ENTRY WorkItem, IN PLIST_ENTRY WorkQueue) { KIRQL OldIrql; PWORK_QUEUE_ITEM ThreadToSpawn; /* First of all, insert the item in the queue */ OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock); InsertTailList(WorkQueue, &WorkItem->WorkQueueLinks); /* Now, define whether we have to spawn a new work thread * We will spawn a new one if: * - There's no throttle in action * - There's still at least one idle thread */ ThreadToSpawn = NULL; if (!CcQueueThrottle && !IsListEmpty(&CcIdleWorkerThreadList)) { PLIST_ENTRY ListEntry; /* Get the idle thread */ ListEntry = RemoveHeadList(&CcIdleWorkerThreadList); ThreadToSpawn = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ITEM, List); /* We're going to have one more! */ CcNumberActiveWorkerThreads += 1; } KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); /* If we have a thread to spawn, do it! */ if (ThreadToSpawn != NULL) { /* We NULLify it to be consistent with initialization */ ThreadToSpawn->List.Flink = NULL; ExQueueWorkItem(ThreadToSpawn, CriticalWorkQueue); } }
/* * @implemented */ NTSTATUS NTAPI CcWaitForCurrentLazyWriterActivity ( VOID) { KIRQL OldIrql; KEVENT WaitEvent; PWORK_QUEUE_ENTRY WorkItem; /* Allocate a work item */ WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList); if (WorkItem == NULL) { return STATUS_INSUFFICIENT_RESOURCES; } /* We want lazy writer to set our event */ WorkItem->Function = SetDone; KeInitializeEvent(&WaitEvent, NotificationEvent, FALSE); WorkItem->Parameters.Event.Event = &WaitEvent; /* Use the post tick queue */ OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); InsertTailList(&CcPostTickWorkQueue, &WorkItem->WorkQueueLinks); /* Inform the lazy writer it will have to handle the post tick queue */ LazyWriter.OtherWork = TRUE; /* And if it's not running, queue a lazy writer run * And start it NOW, we want the response now */ if (!LazyWriter.ScanActive) { CcScheduleLazyWriteScan(TRUE); } KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); /* And now, wait until lazy writer replies */ return KeWaitForSingleObject(&WaitEvent, Executive, KernelMode, FALSE, NULL); }
PMMPTE NTAPI MiReserveAlignedSystemPtes(IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType, IN ULONG Alignment) { KIRQL OldIrql; PMMPTE PreviousPte, NextPte, ReturnPte; ULONG ClusterSize; // // Sanity check // ASSERT(Alignment <= PAGE_SIZE); // // Acquire the System PTE lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueSystemSpaceLock); // // Find the last cluster in the list that doesn't contain enough PTEs // PreviousPte = &MmFirstFreeSystemPte[SystemPtePoolType]; while (PreviousPte->u.List.NextEntry != MM_EMPTY_PTE_LIST) { // // Get the next cluster and its size // NextPte = MmSystemPteBase + PreviousPte->u.List.NextEntry; ClusterSize = MI_GET_CLUSTER_SIZE(NextPte); // // Check if this cluster contains enough PTEs // if (NumberOfPtes <= ClusterSize) break; // // On to the next cluster // PreviousPte = NextPte; } // // Make sure we didn't reach the end of the cluster list // if (PreviousPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) { // // Release the System PTE lock and return failure // KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql); return NULL; } // // Unlink the cluster // PreviousPte->u.List.NextEntry = NextPte->u.List.NextEntry; // // Check if the reservation spans the whole cluster // if (ClusterSize == NumberOfPtes) { // // Return the first PTE of this cluster // ReturnPte = NextPte; // // Zero the cluster // if (NextPte->u.List.OneEntry == 0) { NextPte->u.Long = 0; NextPte++; } NextPte->u.Long = 0; } else { // // Divide the cluster into two parts // ClusterSize -= NumberOfPtes; ReturnPte = NextPte + ClusterSize; // // Set the size of the first cluster, zero the second if needed // if (ClusterSize == 1) { NextPte->u.List.OneEntry = 1; ReturnPte->u.Long = 0; } else { NextPte++; NextPte->u.List.NextEntry = ClusterSize; } // // Step through the cluster list to find out where to insert the first // PreviousPte = &MmFirstFreeSystemPte[SystemPtePoolType]; while (PreviousPte->u.List.NextEntry != MM_EMPTY_PTE_LIST) { // // Get the next cluster // NextPte = MmSystemPteBase + PreviousPte->u.List.NextEntry; // // Check if the cluster to insert is smaller or of equal size // if (ClusterSize <= MI_GET_CLUSTER_SIZE(NextPte)) break; // // On to the next cluster // PreviousPte = NextPte; } // // Retrieve the first cluster and link it back into the cluster list // NextPte = ReturnPte - ClusterSize; NextPte->u.List.NextEntry = PreviousPte->u.List.NextEntry; PreviousPte->u.List.NextEntry = NextPte - MmSystemPteBase; } // // Decrease availability // MmTotalFreeSystemPtes[SystemPtePoolType] -= NumberOfPtes; // // Release the System PTE lock // KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql); // // Flush the TLB // KeFlushProcessTb(); // // Return the reserved PTEs // return ReturnPte; }
VOID NTAPI MiReleaseSystemPtes(IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType) { KIRQL OldIrql; ULONG ClusterSize; PMMPTE PreviousPte, NextPte, InsertPte; // // Check to make sure the PTE address is within bounds // ASSERT(NumberOfPtes != 0); ASSERT(StartingPte >= MmSystemPtesStart[SystemPtePoolType]); ASSERT(StartingPte + NumberOfPtes - 1 <= MmSystemPtesEnd[SystemPtePoolType]); // // Zero PTEs // RtlZeroMemory(StartingPte, NumberOfPtes * sizeof(MMPTE)); // // Acquire the System PTE lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueSystemSpaceLock); // // Increase availability // MmTotalFreeSystemPtes[SystemPtePoolType] += NumberOfPtes; // // Step through the cluster list to find where to insert the PTEs // PreviousPte = &MmFirstFreeSystemPte[SystemPtePoolType]; InsertPte = NULL; while (PreviousPte->u.List.NextEntry != MM_EMPTY_PTE_LIST) { // // Get the next cluster and its size // NextPte = MmSystemPteBase + PreviousPte->u.List.NextEntry; ClusterSize = MI_GET_CLUSTER_SIZE(NextPte); // // Check if this cluster is adjacent to the PTEs being released // if ((NextPte + ClusterSize == StartingPte) || (StartingPte + NumberOfPtes == NextPte)) { // // Add the PTEs in the cluster to the PTEs being released // NumberOfPtes += ClusterSize; if (NextPte < StartingPte) StartingPte = NextPte; // // Unlink this cluster and zero it // PreviousPte->u.List.NextEntry = NextPte->u.List.NextEntry; if (NextPte->u.List.OneEntry == 0) { NextPte->u.Long = 0; NextPte++; } NextPte->u.Long = 0; // // Invalidate the previously found insertion location, if any // InsertPte = NULL; } else { // // Check if the insertion location is right before this cluster // if ((InsertPte == NULL) && (NumberOfPtes <= ClusterSize)) InsertPte = PreviousPte; // // On to the next cluster // PreviousPte = NextPte; } } // // If no insertion location was found, use the tail of the list // if (InsertPte == NULL) InsertPte = PreviousPte; // // Create a new cluster using the PTEs being released // if (NumberOfPtes != 1) { StartingPte->u.List.OneEntry = 0; NextPte = StartingPte + 1; NextPte->u.List.NextEntry = NumberOfPtes; } else StartingPte->u.List.OneEntry = 1; // // Link the new cluster into the cluster list at the insertion location // StartingPte->u.List.NextEntry = InsertPte->u.List.NextEntry; InsertPte->u.List.NextEntry = StartingPte - MmSystemPteBase; // // Release the System PTE lock // KeReleaseQueuedSpinLock(LockQueueSystemSpaceLock, OldIrql); }
NTSTATUS NTAPI MmNotPresentFaultCachePage ( _In_ PMMSUPPORT AddressSpace, _In_ MEMORY_AREA* MemoryArea, _In_ PVOID Address, _In_ BOOLEAN Locked, _Inout_ PMM_REQUIRED_RESOURCES Required) { NTSTATUS Status; PVOID PAddress; ULONG Consumer; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER FileOffset, TotalOffset; ULONG_PTR Entry; ULONG Attributes; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); KIRQL OldIrql; DPRINT("Not Present: %p %p (%p-%p)\n", AddressSpace, Address, MemoryArea->StartingAddress, MemoryArea->EndingAddress); /* * There is a window between taking the page fault and locking the * address space when another thread could load the page so we check * that. */ if (MmIsPagePresent(Process, Address)) { DPRINT("Done\n"); return STATUS_SUCCESS; } PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); TotalOffset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress; Segment = MemoryArea->Data.SectionData.Segment; TotalOffset.QuadPart += MemoryArea->Data.SectionData.ViewOffset.QuadPart; FileOffset = TotalOffset; //Consumer = (Segment->Flags & MM_DATAFILE_SEGMENT) ? MC_CACHE : MC_USER; Consumer = MC_CACHE; if (Segment->FileObject) { DPRINT("FileName %wZ\n", &Segment->FileObject->FileName); } DPRINT("Total Offset %08x%08x\n", TotalOffset.HighPart, TotalOffset.LowPart); /* Lock the segment */ MmLockSectionSegment(Segment); /* Get the entry corresponding to the offset within the section */ Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset); Attributes = PAGE_READONLY; if (Required->State && Required->Page[0]) { DPRINT("Have file and page, set page %x in section @ %x #\n", Required->Page[0], TotalOffset.LowPart); if (Required->SwapEntry) MmSetSavedSwapEntryPage(Required->Page[0], Required->SwapEntry); if (Required->State & 2) { DPRINT("Set in section @ %x\n", TotalOffset.LowPart); Status = MmSetPageEntrySectionSegment(Segment, &TotalOffset, Entry = MAKE_PFN_SSE(Required->Page[0])); if (!NT_SUCCESS(Status)) { MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); } MmUnlockSectionSegment(Segment); MiSetPageEvent(Process, Address); DPRINT("Status %x\n", Status); return STATUS_MM_RESTART_OPERATION; } else { DPRINT("Set %x in address space @ %p\n", Required->Page[0], Address); Status = MmCreateVirtualMapping(Process, Address, Attributes, Required->Page, 1); if (NT_SUCCESS(Status)) { MmInsertRmap(Required->Page[0], Process, Address); } else { /* Drop the reference for our address space ... */ MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); } MmUnlockSectionSegment(Segment); DPRINTC("XXX Set Event %x\n", Status); MiSetPageEvent(Process, Address); DPRINT("Status %x\n", Status); return Status; } } else if (MM_IS_WAIT_PTE(Entry)) { // Whenever MM_WAIT_ENTRY is required as a swap entry, we need to // ask the fault handler to wait until we should continue. Rathern // than recopy this boilerplate code everywhere, we just ask them // to wait. MmUnlockSectionSegment(Segment); return STATUS_SUCCESS + 1; } else if (Entry) { PFN_NUMBER Page = PFN_FROM_SSE(Entry); DPRINT("Take reference to page %x #\n", Page); if (MiGetPfnEntry(Page) == NULL) { DPRINT1("Found no PFN entry for page 0x%x in page entry 0x%x (segment: 0x%p, offset: %08x%08x)\n", Page, Entry, Segment, TotalOffset.HighPart, TotalOffset.LowPart); KeBugCheck(CACHE_MANAGER); } OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MmReferencePage(Page); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); Status = MmCreateVirtualMapping(Process, Address, Attributes, &Page, 1); if (NT_SUCCESS(Status)) { MmInsertRmap(Page, Process, Address); } DPRINT("XXX Set Event %x\n", Status); MiSetPageEvent(Process, Address); MmUnlockSectionSegment(Segment); DPRINT("Status %x\n", Status); return Status; } else { DPRINT("Get page into section\n"); /* * If the entry is zero (and it can't change because we have * locked the segment) then we need to load the page. */ //DPRINT1("Read from file %08x %wZ\n", FileOffset.LowPart, &Section->FileObject->FileName); Required->State = 2; Required->Context = Segment->FileObject; Required->Consumer = Consumer; Required->FileOffset = FileOffset; Required->Amount = PAGE_SIZE; Required->DoAcquisition = MiReadFilePage; MmSetPageEntrySectionSegment(Segment, &TotalOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); MmUnlockSectionSegment(Segment); return STATUS_MORE_PROCESSING_REQUIRED; } ASSERT(FALSE); return STATUS_ACCESS_VIOLATION; }
BOOLEAN NTAPI MmCreateProcessAddressSpace(IN ULONG MinWs, IN PEPROCESS Process, OUT PULONG_PTR DirectoryTableBase) { KIRQL OldIrql; PFN_NUMBER TableBasePfn, HyperPfn, HyperPdPfn, HyperPtPfn, WorkingSetPfn; PMMPTE SystemPte; MMPTE TempPte, PdePte; ULONG TableIndex; PMMPTE PageTablePointer; /* Make sure we don't already have a page directory setup */ ASSERT(Process->Pcb.DirectoryTableBase[0] == 0); ASSERT(Process->Pcb.DirectoryTableBase[1] == 0); ASSERT(Process->WorkingSetPage == 0); /* Choose a process color */ Process->NextPageColor = (USHORT)RtlRandom(&MmProcessColorSeed); /* Setup the hyperspace lock */ KeInitializeSpinLock(&Process->HyperSpaceLock); /* Lock PFN database */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Get a page for the table base and one for hyper space. The PFNs for these pages will be initialized in MmInitializeProcessAddressSpace, when we are already attached to the process. */ TableBasePfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); HyperPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); HyperPdPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); HyperPtPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); WorkingSetPfn = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(Process)); /* Release PFN lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); /* Zero pages */ /// FIXME: MiZeroPhysicalPage(HyperPfn); MiZeroPhysicalPage(WorkingSetPfn); /* Set the base directory pointers */ Process->WorkingSetPage = WorkingSetPfn; DirectoryTableBase[0] = TableBasePfn << PAGE_SHIFT; DirectoryTableBase[1] = HyperPfn << PAGE_SHIFT; /* Get a PTE to map the page directory */ SystemPte = MiReserveSystemPtes(1, SystemPteSpace); ASSERT(SystemPte != NULL); /* Get its address */ PageTablePointer = MiPteToAddress(SystemPte); /* Build the PTE for the page directory and map it */ PdePte = ValidKernelPte; PdePte.u.Hard.PageFrameNumber = TableBasePfn; *SystemPte = PdePte; /// architecture specific //MiInitializePageDirectoryForProcess( /* Copy the kernel mappings and zero out the rest */ TableIndex = PXE_PER_PAGE / 2; RtlZeroMemory(PageTablePointer, TableIndex * sizeof(MMPTE)); RtlCopyMemory(PageTablePointer + TableIndex, MiAddressToPxe(0) + TableIndex, PAGE_SIZE - TableIndex * sizeof(MMPTE)); /* Sanity check */ ASSERT(MiAddressToPxi(MmHyperSpaceEnd) >= TableIndex); /* Setup a PTE for the page directory mappings */ TempPte = ValidKernelPte; /* Update the self mapping of the PML4 */ TableIndex = MiAddressToPxi((PVOID)PXE_SELFMAP); TempPte.u.Hard.PageFrameNumber = TableBasePfn; PageTablePointer[TableIndex] = TempPte; /* Write the PML4 entry for hyperspace */ TableIndex = MiAddressToPxi((PVOID)HYPER_SPACE); TempPte.u.Hard.PageFrameNumber = HyperPfn; PageTablePointer[TableIndex] = TempPte; /* Map the hyperspace PDPT to the system PTE */ PdePte.u.Hard.PageFrameNumber = HyperPfn; *SystemPte = PdePte; __invlpg(PageTablePointer); /* Write the hyperspace entry for the first PD */ TempPte.u.Hard.PageFrameNumber = HyperPdPfn; PageTablePointer[0] = TempPte; /* Map the hyperspace PD to the system PTE */ PdePte.u.Hard.PageFrameNumber = HyperPdPfn; *SystemPte = PdePte; __invlpg(PageTablePointer); /* Write the hyperspace entry for the first PT */ TempPte.u.Hard.PageFrameNumber = HyperPtPfn; PageTablePointer[0] = TempPte; /* Map the hyperspace PT to the system PTE */ PdePte.u.Hard.PageFrameNumber = HyperPtPfn; *SystemPte = PdePte; __invlpg(PageTablePointer); /* Write the hyperspace PTE for the working set list index */ TempPte.u.Hard.PageFrameNumber = WorkingSetPfn; TableIndex = MiAddressToPti(MmWorkingSetList); PageTablePointer[TableIndex] = TempPte; /// end architecture specific /* Release the system PTE */ MiReleaseSystemPtes(SystemPte, 1, SystemPteSpace); /* Switch to phase 1 initialization */ ASSERT(Process->AddressSpaceInitialized == 0); Process->AddressSpaceInitialized = 1; return TRUE; }
VOID NTAPI MmFreeSpecialPool(PVOID P) { PMMPTE PointerPte; PPOOL_HEADER Header; BOOLEAN Overruns = FALSE; KIRQL Irql = KeGetCurrentIrql(); POOL_TYPE PoolType; ULONG BytesRequested, BytesReal = 0; ULONG PtrOffset; PUCHAR b; PMI_FREED_SPECIAL_POOL FreedHeader; LARGE_INTEGER TickCount; PMMPFN Pfn; DPRINT1("MmFreeSpecialPool(%p)\n", P); /* Get the PTE */ PointerPte = MiAddressToPte(P); /* Check if it's valid */ if (PointerPte->u.Hard.Valid == 0) { /* Bugcheck if it has NOACCESS or 0 set as protection */ if (PointerPte->u.Soft.Protection == MM_NOACCESS || !PointerPte->u.Soft.Protection) { KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, (ULONG_PTR)PointerPte, 0, 0x20); } } /* Determine if it's a underruns or overruns pool pointer */ PtrOffset = (ULONG)((ULONG_PTR)P & (PAGE_SIZE - 1)); if (PtrOffset) { /* Pool catches overruns */ Header = PAGE_ALIGN(P); Overruns = TRUE; } else { /* Pool catches underruns */ Header = (PPOOL_HEADER)((PUCHAR)PAGE_ALIGN(P) + PAGE_SIZE - sizeof(POOL_HEADER)); } /* Check if it's non paged pool */ if ((Header->Ulong1 & SPECIAL_POOL_PAGED) == 0) { /* Non-paged allocation, ensure that IRQ is not higher that DISPATCH */ ASSERT((PointerPte + 1)->u.Soft.PageFileHigh == SPECIAL_POOL_NONPAGED_PTE); if (Irql > DISPATCH_LEVEL) { KeBugCheckEx(BAD_POOL_HEADER, Irql, (ULONG_PTR)P, 0, 0x31); } PoolType = NonPagedPool; } else { /* Paged allocation, ensure */ ASSERT((PointerPte + 1)->u.Soft.PageFileHigh == SPECIAL_POOL_PAGED_PTE); if (Irql > DISPATCH_LEVEL) { KeBugCheckEx(BAD_POOL_HEADER, Irql, (ULONG_PTR)P, 1, 0x31); } PoolType = PagedPool; } /* Get amount of bytes user requested to be allocated by clearing out the paged mask */ BytesRequested = (Header->Ulong1 & ~SPECIAL_POOL_PAGED) & 0xFFFF; /* Check memory before the allocated user buffer in case of overruns detection */ if (Overruns) { /* Calculate the real placement of the buffer */ BytesReal = PAGE_SIZE - PtrOffset; /* If they mismatch, it's unrecoverable */ if (BytesRequested > BytesReal) { KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, BytesRequested, BytesReal, 0x21); } if (BytesRequested + sizeof(POOL_HEADER) < BytesReal) { KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, BytesRequested, BytesReal, 0x22); } /* Actually check the memory pattern */ for (b = (PUCHAR)(Header + 1); b < (PUCHAR)P; b++) { if (Header->BlockSize != b[0]) { /* Bytes mismatch */ KeBugCheckEx(BAD_POOL_HEADER, (ULONG_PTR)P, (ULONG_PTR)b, Header->BlockSize, 0x23); } } } /* Check the memory pattern after the user buffer */ MiSpecialPoolCheckPattern(P, Header); /* Fill the freed header */ KeQueryTickCount(&TickCount); FreedHeader = (PMI_FREED_SPECIAL_POOL)PAGE_ALIGN(P); FreedHeader->Signature = 0x98764321; FreedHeader->TickCount = TickCount.LowPart; FreedHeader->NumberOfBytesRequested = BytesRequested; FreedHeader->Pagable = PoolType; FreedHeader->VirtualAddress = P; FreedHeader->Thread = PsGetCurrentThread(); /* TODO: Fill StackPointer and StackBytes */ FreedHeader->StackPointer = NULL; FreedHeader->StackBytes = 0; if (PoolType == NonPagedPool) { /* Non pagable. Get PFN element corresponding to the PTE */ Pfn = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber); /* Lock PFN database */ ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL); Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Delete this PFN */ MI_SET_PFN_DELETED(Pfn); /* Decrement share count of this PFN */ MiDecrementShareCount(Pfn, PointerPte->u.Hard.PageFrameNumber); /* Flush the TLB */ //FIXME: Use KeFlushSingleTb() instead KeFlushEntireTb(TRUE, TRUE); } else { /* Pagable. Delete that virtual address */ MiDeleteSystemPageableVm(PointerPte, 1, 0, NULL); /* Lock PFN database */ ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL); Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } /* Mark next PTE as invalid */ PointerPte[1].u.Long = 0; //|= 8000; /* Make sure that the last entry is really the last one */ ASSERT(MiSpecialPoolLastPte->u.List.NextEntry == MM_EMPTY_PTE_LIST); /* Update the current last PTE next pointer */ MiSpecialPoolLastPte->u.List.NextEntry = PointerPte - MmSystemPteBase; /* PointerPte becomes the new last PTE */ PointerPte->u.List.NextEntry = MM_EMPTY_PTE_LIST; MiSpecialPoolLastPte = PointerPte; /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); }
VOID NTAPI INIT_FUNCTION MiInitializePoolEvents(VOID) { KIRQL OldIrql; PFN_NUMBER FreePoolInPages; /* Lock paged pool */ KeAcquireGuardedMutex(&MmPagedPoolMutex); /* Total size of the paged pool minus the allocated size, is free */ FreePoolInPages = MmSizeOfPagedPoolInPages - MmPagedPoolInfo.AllocatedPagedPool; /* Check the initial state high state */ if (FreePoolInPages >= MiHighPagedPoolThreshold) { /* We have plenty of pool */ KeSetEvent(MiHighPagedPoolEvent, 0, FALSE); } else { /* We don't */ KeClearEvent(MiHighPagedPoolEvent); } /* Check the initial low state */ if (FreePoolInPages <= MiLowPagedPoolThreshold) { /* We're very low in free pool memory */ KeSetEvent(MiLowPagedPoolEvent, 0, FALSE); } else { /* We're not */ KeClearEvent(MiLowPagedPoolEvent); } /* Release the paged pool lock */ KeReleaseGuardedMutex(&MmPagedPoolMutex); /* Now it's time for the nonpaged pool lock */ OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); /* Free pages are the maximum minus what's been allocated */ FreePoolInPages = MmMaximumNonPagedPoolInPages - MmAllocatedNonPagedPool; /* Check if we have plenty */ if (FreePoolInPages >= MiHighNonPagedPoolThreshold) { /* We do, set the event */ KeSetEvent(MiHighNonPagedPoolEvent, 0, FALSE); } else { /* We don't, clear the event */ KeClearEvent(MiHighNonPagedPoolEvent); } /* Check if we have very little */ if (FreePoolInPages <= MiLowNonPagedPoolThreshold) { /* We do, set the event */ KeSetEvent(MiLowNonPagedPoolEvent, 0, FALSE); } else { /* We don't, clear it */ KeClearEvent(MiLowNonPagedPoolEvent); } /* We're done, release the nonpaged pool lock */ KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); }
/* * @implemented */ VOID NTAPI MmFreePagesFromMdl(IN PMDL Mdl) { PVOID Base; PPFN_NUMBER Pages; LONG NumberOfPages; PMMPFN Pfn1; KIRQL OldIrql; DPRINT("Freeing MDL: %p\n", Mdl); // // Sanity checks // ASSERT(KeGetCurrentIrql() <= APC_LEVEL); ASSERT((Mdl->MdlFlags & MDL_IO_SPACE) == 0); ASSERT(((ULONG_PTR)Mdl->StartVa & (PAGE_SIZE - 1)) == 0); // // Get address and page information // Base = (PVOID)((ULONG_PTR)Mdl->StartVa + Mdl->ByteOffset); NumberOfPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Base, Mdl->ByteCount); // // Acquire PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); // // Loop all the MDL pages // Pages = (PPFN_NUMBER)(Mdl + 1); do { // // Reached the last page // if (*Pages == LIST_HEAD) break; // // Get the page entry // Pfn1 = MiGetPfnEntry(*Pages); ASSERT(Pfn1); ASSERT(Pfn1->u2.ShareCount == 1); ASSERT(MI_IS_PFN_DELETED(Pfn1) == TRUE); if (Pfn1->u4.PteFrame != 0x1FFEDCB) { /* Corrupted PFN entry or invalid free */ KeBugCheckEx(MEMORY_MANAGEMENT, 0x1236, (ULONG_PTR)Mdl, (ULONG_PTR)Pages, *Pages); } // // Clear it // Pfn1->u3.e1.StartOfAllocation = 0; Pfn1->u3.e1.EndOfAllocation = 0; Pfn1->u2.ShareCount = 0; // // Dereference it // ASSERT(Pfn1->u3.e2.ReferenceCount != 0); if (Pfn1->u3.e2.ReferenceCount != 1) { /* Just take off one reference */ InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); } else { /* We'll be nuking the whole page */ MiDecrementReferenceCount(Pfn1, *Pages); } // // Clear this page and move on // *Pages++ = LIST_HEAD; } while (--NumberOfPages != 0); // // Release the lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // Remove the pages locked flag // Mdl->MdlFlags &= ~MDL_PAGES_LOCKED; }
VOID NTAPI MiBalancerThread(PVOID Unused) { PVOID WaitObjects[2]; NTSTATUS Status; ULONG i; WaitObjects[0] = &MiBalancerEvent; WaitObjects[1] = &MiBalancerTimer; while (1) { Status = KeWaitForMultipleObjects(2, WaitObjects, WaitAny, Executive, KernelMode, FALSE, NULL, NULL); if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1) { ULONG InitialTarget = 0; #if (_MI_PAGING_LEVELS == 2) if (!MiIsBalancerThread()) { /* Clean up the unused PDEs */ ULONG_PTR Address; PEPROCESS Process = PsGetCurrentProcess(); /* Acquire PFN lock */ KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); PMMPDE pointerPde; for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS; Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS; Address += (PAGE_SIZE * PTE_COUNT)) { if (MiQueryPageTableReferences((PVOID)Address) == 0) { pointerPde = MiAddressToPde(Address); if (pointerPde->u.Hard.Valid) MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL); ASSERT(pointerPde->u.Hard.Valid == 0); } } /* Release lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } #endif do { ULONG OldTarget = InitialTarget; /* Trim each consumer */ for (i = 0; i < MC_MAXIMUM; i++) { InitialTarget = MiTrimMemoryConsumer(i, InitialTarget); } /* No pages left to swap! */ if (InitialTarget != 0 && InitialTarget == OldTarget) { /* Game over */ KeBugCheck(NO_PAGES_AVAILABLE); } } while (InitialTarget != 0); } else { DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status); KeBugCheck(MEMORY_MANAGEMENT); } } }
VOID CcLazyWriteScan(VOID) { ULONG Target; KIRQL OldIrql; PLIST_ENTRY ListEntry; LIST_ENTRY ToPost; PWORK_QUEUE_ENTRY WorkItem; /* Do we have entries to queue after we're done? */ InitializeListHead(&ToPost); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); if (LazyWriter.OtherWork) { while (!IsListEmpty(&CcPostTickWorkQueue)) { ListEntry = RemoveHeadList(&CcPostTickWorkQueue); WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks); InsertTailList(&ToPost, &WorkItem->WorkQueueLinks); } LazyWriter.OtherWork = FALSE; } KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); /* Our target is one-eighth of the dirty pages */ Target = CcTotalDirtyPages / 8; if (Target != 0) { /* There is stuff to flush, schedule a write-behind operation */ /* Allocate a work item */ WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList); if (WorkItem != NULL) { WorkItem->Function = WriteBehind; CcPostWorkQueue(WorkItem, &CcRegularWorkQueue); } } /* Post items that were due for end of run */ while (!IsListEmpty(&ToPost)) { ListEntry = RemoveHeadList(&ToPost); WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks); CcPostWorkQueue(WorkItem, &CcRegularWorkQueue); } /* If we have deferred writes, try them now! */ if (!IsListEmpty(&CcDeferredWrites)) { CcPostDeferredWrites(); /* Reschedule immediately a lazy writer run * Keep us active to have short idle delay */ CcScheduleLazyWriteScan(FALSE); } else { /* We're no longer active */ OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); LazyWriter.ScanActive = FALSE; KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); } }
VOID NTAPI CcWorkerThread( IN PVOID Parameter) { KIRQL OldIrql; BOOLEAN DropThrottle, WritePerformed; PWORK_QUEUE_ITEM Item; #if DBG PIRP TopLevel; #endif /* Get back our thread item */ Item = Parameter; /* And by default, don't touch throttle */ DropThrottle = FALSE; /* No write performed */ WritePerformed = FALSE; #if DBG /* Top level IRP should be clean when started * Save it to catch buggy drivers (or bugs!) */ TopLevel = IoGetTopLevelIrp(); if (TopLevel != NULL) { DPRINT1("(%p) TopLevel IRP for this thread: %p\n", PsGetCurrentThread(), TopLevel); } #endif /* Loop till we have jobs */ while (TRUE) { PWORK_QUEUE_ENTRY WorkItem; /* Lock queues */ OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock); /* If we have to touch throttle, reset it now! */ if (DropThrottle) { CcQueueThrottle = FALSE; DropThrottle = FALSE; } /* Check first if we have read ahead to do */ if (IsListEmpty(&CcExpressWorkQueue)) { /* If not, check regular queue */ if (IsListEmpty(&CcRegularWorkQueue)) { break; } else { WorkItem = CONTAINING_RECORD(CcRegularWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks); } } else { WorkItem = CONTAINING_RECORD(CcExpressWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks); } /* Get our work item, if someone is waiting for us to finish * and we're not the only thread in queue * then, quit running to let the others do * and throttle so that noone starts till current activity is over */ if (WorkItem->Function == SetDone && CcNumberActiveWorkerThreads > 1) { CcQueueThrottle = TRUE; break; } /* Otherwise, remove current entry */ RemoveEntryList(&WorkItem->WorkQueueLinks); KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); /* And handle it */ switch (WorkItem->Function) { case ReadAhead: CcPerformReadAhead(WorkItem->Parameters.Read.FileObject); break; case WriteBehind: PsGetCurrentThread()->MemoryMaker = 1; CcWriteBehind(); PsGetCurrentThread()->MemoryMaker = 0; WritePerformed = TRUE; break; case LazyScan: CcLazyWriteScan(); break; case SetDone: KeSetEvent(WorkItem->Parameters.Event.Event, IO_NO_INCREMENT, FALSE); DropThrottle = TRUE; break; default: DPRINT1("Ignored item: %p (%d)\n", WorkItem, WorkItem->Function); break; } /* And release the item */ ExFreeToNPagedLookasideList(&CcTwilightLookasideList, WorkItem); } /* Our thread is available again */ InsertTailList(&CcIdleWorkerThreadList, &Item->List); /* One less worker */ --CcNumberActiveWorkerThreads; KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql); /* If there are pending write openations and we have at least 20 dirty pages */ if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20) { /* And if we performed a write operation previously, then * stress the system a bit and reschedule a scan to find * stuff to write */ if (WritePerformed) { CcLazyWriteScan(); } } #if DBG /* Top level shouldn't have changed */ if (TopLevel != IoGetTopLevelIrp()) { DPRINT1("(%p) Mismatching TopLevel: %p, %p\n", PsGetCurrentThread(), TopLevel, IoGetTopLevelIrp()); } #endif }
NTSTATUS NTAPI MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, PPFN_NUMBER AllocatedPage) { ULONG OldUsed; PFN_NUMBER Page; KIRQL OldIrql; /* * Make sure we don't exceed our individual target. */ OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) && !MiIsBalancerThread()) { if (!CanWait) { (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); return(STATUS_NO_MEMORY); } MiTrimMemoryConsumer(Consumer); } /* * Allocate always memory for the non paged pool and for the pager thread. */ if ((Consumer == MC_SYSTEM) || MiIsBalancerThread()) { OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); Page = MmAllocPage(Consumer); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); if (Page == 0) { KeBugCheck(NO_PAGES_AVAILABLE); } *AllocatedPage = Page; if (MmAvailablePages <= MiMinimumAvailablePages && MiBalancerThreadHandle != NULL) { KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE); } return(STATUS_SUCCESS); } /* * Make sure we don't exceed global targets. */ if (MmAvailablePages <= MiMinimumAvailablePages) { MM_ALLOCATION_REQUEST Request; if (!CanWait) { (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); return(STATUS_NO_MEMORY); } /* Insert an allocation request. */ Request.Page = 0; KeInitializeEvent(&Request.Event, NotificationEvent, FALSE); (void)InterlockedIncrementUL(&MiPagesRequired); KeAcquireSpinLock(&AllocationListLock, &OldIrql); if (MiBalancerThreadHandle != NULL) { KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE); } InsertTailList(&AllocationListHead, &Request.ListEntry); KeReleaseSpinLock(&AllocationListLock, OldIrql); KeWaitForSingleObject(&Request.Event, 0, KernelMode, FALSE, NULL); Page = Request.Page; if (Page == 0) { KeBugCheck(NO_PAGES_AVAILABLE); } /* Update the Consumer and make the page active */ if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; (void)InterlockedDecrementUL(&MiPagesRequired); return(STATUS_SUCCESS); } /* * Actually allocate the page. */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); Page = MmAllocPage(Consumer); KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); if (Page == 0) { KeBugCheck(NO_PAGES_AVAILABLE); } if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; return(STATUS_SUCCESS); }
VOID NTAPI MmZeroPageThread(VOID) { PKTHREAD Thread = KeGetCurrentThread(); PVOID StartAddress, EndAddress; PVOID WaitObjects[2]; KIRQL OldIrql; PVOID ZeroAddress; PFN_NUMBER PageIndex, FreePage; PMMPFN Pfn1; /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); /* Set our priority to 0 */ Thread->BasePriority = 0; KeSetPriorityThread(Thread, 0); /* Setup the wait objects */ WaitObjects[0] = &MmZeroingPageEvent; // WaitObjects[1] = &PoSystemIdleTimer; FIXME: Implement idle timer while (TRUE) { KeWaitForMultipleObjects(1, // 2 WaitObjects, WaitAny, WrFreePage, KernelMode, FALSE, NULL, NULL); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); while (TRUE) { if (!MmFreePageListHead.Total) { MmZeroingPageThreadActive = FALSE; KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); break; } PageIndex = MmFreePageListHead.Flink; ASSERT(PageIndex != LIST_HEAD); Pfn1 = MiGetPfnEntry(PageIndex); MI_SET_USAGE(MI_USAGE_ZERO_LOOP); MI_SET_PROCESS2("Kernel 0 Loop"); FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex)); /* The first global free page should also be the first on its own list */ if (FreePage != PageIndex) { KeBugCheckEx(PFN_LIST_CORRUPT, 0x8F, FreePage, PageIndex, 0); } Pfn1->u1.Flink = LIST_HEAD; KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); ZeroAddress = MiMapPagesInZeroSpace(Pfn1, 1); ASSERT(ZeroAddress); RtlZeroMemory(ZeroAddress, PAGE_SIZE); MiUnmapPagesInZeroSpace(ZeroAddress, 1); OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); MiInsertPageInList(&MmZeroedPageListHead, PageIndex); } } }
PFN_NUMBER NTAPI MiFindContiguousPages(IN PFN_NUMBER LowestPfn, IN PFN_NUMBER HighestPfn, IN PFN_NUMBER BoundaryPfn, IN PFN_NUMBER SizeInPages, IN MEMORY_CACHING_TYPE CacheType) { PFN_NUMBER Page, PageCount, LastPage, Length, BoundaryMask; ULONG i = 0; PMMPFN Pfn1, EndPfn; KIRQL OldIrql; PAGED_CODE(); ASSERT(SizeInPages != 0); // // Convert the boundary PFN into an alignment mask // BoundaryMask = ~(BoundaryPfn - 1); /* Disable APCs */ KeEnterGuardedRegion(); // // Loop all the physical memory blocks // do { // // Capture the base page and length of this memory block // Page = MmPhysicalMemoryBlock->Run[i].BasePage; PageCount = MmPhysicalMemoryBlock->Run[i].PageCount; // // Check how far this memory block will go // LastPage = Page + PageCount; // // Trim it down to only the PFNs we're actually interested in // if ((LastPage - 1) > HighestPfn) LastPage = HighestPfn + 1; if (Page < LowestPfn) Page = LowestPfn; // // Skip this run if it's empty or fails to contain all the pages we need // if (!(PageCount) || ((Page + SizeInPages) > LastPage)) continue; // // Now scan all the relevant PFNs in this run // Length = 0; for (Pfn1 = MI_PFN_ELEMENT(Page); Page < LastPage; Page++, Pfn1++) { // // If this PFN is in use, ignore it // if (MiIsPfnInUse(Pfn1)) { Length = 0; continue; } // // If we haven't chosen a start PFN yet and the caller specified an // alignment, make sure the page matches the alignment restriction // if ((!(Length) && (BoundaryPfn)) && (((Page ^ (Page + SizeInPages - 1)) & BoundaryMask))) { // // It does not, so bail out // continue; } // // Increase the number of valid pages, and check if we have enough // if (++Length == SizeInPages) { // // It appears we've amassed enough legitimate pages, rollback // Pfn1 -= (Length - 1); Page -= (Length - 1); // // Acquire the PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // Things might've changed for us. Is the page still free? // if (MiIsPfnInUse(Pfn1)) break; // // So far so good. Is this the last confirmed valid page? // if (!--Length) { // // Sanity check that we didn't go out of bounds // ASSERT(i != MmPhysicalMemoryBlock->NumberOfRuns); // // Loop until all PFN entries have been processed // EndPfn = Pfn1 - SizeInPages + 1; do { // // This PFN is now a used page, set it up // MI_SET_USAGE(MI_USAGE_CONTINOUS_ALLOCATION); MI_SET_PROCESS2("Kernel Driver"); MiUnlinkFreeOrZeroedPage(Pfn1); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u3.e1.StartOfAllocation = 0; Pfn1->u3.e1.EndOfAllocation = 0; Pfn1->u3.e1.PrototypePte = 0; Pfn1->u4.VerifierAllocation = 0; Pfn1->PteAddress = (PVOID)0xBAADF00D; // // Check if this is the last PFN, otherwise go on // if (Pfn1 == EndPfn) break; Pfn1--; } while (TRUE); // // Mark the first and last PFN so we can find them later // Pfn1->u3.e1.StartOfAllocation = 1; (Pfn1 + SizeInPages - 1)->u3.e1.EndOfAllocation = 1; // // Now it's safe to let go of the PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // Quick sanity check that the last PFN is consistent // EndPfn = Pfn1 + SizeInPages; ASSERT(EndPfn == MI_PFN_ELEMENT(Page + 1)); // // Compute the first page, and make sure it's consistent // Page = Page - SizeInPages + 1; ASSERT(Pfn1 == MI_PFN_ELEMENT(Page)); ASSERT(Page != 0); /* Enable APCs and return the page */ KeLeaveGuardedRegion(); return Page; } // // Keep going. The purpose of this loop is to reconfirm that // after acquiring the PFN lock these pages are still usable // Pfn1++; Page++; } while (TRUE); // // If we got here, something changed while we hadn't acquired // the PFN lock yet, so we'll have to restart // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); Length = 0; } } } while (++i != MmPhysicalMemoryBlock->NumberOfRuns); // // And if we get here, it means no suitable physical memory runs were found // return 0; }
/* * @implemented */ VOID NTAPI MmUnlockPages(IN PMDL Mdl) { PPFN_NUMBER MdlPages, LastPage; PEPROCESS Process; PVOID Base; ULONG Flags, PageCount; KIRQL OldIrql; PMMPFN Pfn1; DPRINT("Unlocking MDL: %p\n", Mdl); // // Sanity checks // ASSERT((Mdl->MdlFlags & MDL_PAGES_LOCKED) != 0); ASSERT((Mdl->MdlFlags & MDL_SOURCE_IS_NONPAGED_POOL) == 0); ASSERT((Mdl->MdlFlags & MDL_PARTIAL) == 0); ASSERT(Mdl->ByteCount != 0); // // Get the process associated and capture the flags which are volatile // Process = Mdl->Process; Flags = Mdl->MdlFlags; // // Automagically undo any calls to MmGetSystemAddressForMdl's for this MDL // if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { // // Unmap the pages from system space // MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl); } // // Get the page count // MdlPages = (PPFN_NUMBER)(Mdl + 1); Base = (PVOID)((ULONG_PTR)Mdl->StartVa + Mdl->ByteOffset); PageCount = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Base, Mdl->ByteCount); ASSERT(PageCount != 0); // // We don't support AWE // if (Flags & MDL_DESCRIBES_AWE) ASSERT(FALSE); // // Check if the buffer is mapped I/O space // if (Flags & MDL_IO_SPACE) { // // Acquire PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); // // Loop every page // LastPage = MdlPages + PageCount; do { // // Last page, break out // if (*MdlPages == LIST_HEAD) break; // // Check if this page is in the PFN database // Pfn1 = MiGetPfnEntry(*MdlPages); if (Pfn1) MiDereferencePfnAndDropLockCount(Pfn1); } while (++MdlPages < LastPage); // // Release the lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // Check if we have a process // if (Process) { // // Handle the accounting of locked pages // ASSERT(Process->NumberOfLockedPages > 0); InterlockedExchangeAddSizeT(&Process->NumberOfLockedPages, -(LONG_PTR)PageCount); } // // We're done // Mdl->MdlFlags &= ~MDL_IO_SPACE; Mdl->MdlFlags &= ~MDL_PAGES_LOCKED; return; } // // Check if we have a process // if (Process) { // // Handle the accounting of locked pages // ASSERT(Process->NumberOfLockedPages > 0); InterlockedExchangeAddSizeT(&Process->NumberOfLockedPages, -(LONG_PTR)PageCount); } // // Loop every page // LastPage = MdlPages + PageCount; do { // // Last page reached // if (*MdlPages == LIST_HEAD) { // // Were there no pages at all? // if (MdlPages == (PPFN_NUMBER)(Mdl + 1)) { // // We're already done // Mdl->MdlFlags &= ~MDL_PAGES_LOCKED; return; } // // Otherwise, stop here // LastPage = MdlPages; break; } /* Save the PFN entry instead for the secondary loop */ *MdlPages = (PFN_NUMBER)MiGetPfnEntry(*MdlPages); ASSERT(*MdlPages != 0); } while (++MdlPages < LastPage); // // Reset pointer // MdlPages = (PPFN_NUMBER)(Mdl + 1); // // Now grab the PFN lock for the actual unlock and dereference // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { /* Get the current entry and reference count */ Pfn1 = (PMMPFN)*MdlPages; MiDereferencePfnAndDropLockCount(Pfn1); } while (++MdlPages < LastPage); // // Release the lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // We're done // Mdl->MdlFlags &= ~MDL_PAGES_LOCKED; }
VOID NTAPI MiFreeContiguousMemory(IN PVOID BaseAddress) { KIRQL OldIrql; PFN_NUMBER PageFrameIndex, LastPage, PageCount; PMMPFN Pfn1, StartPfn; PMMPTE PointerPte; PAGED_CODE(); // // First, check if the memory came from initial nonpaged pool, or expansion // if (((BaseAddress >= MmNonPagedPoolStart) && (BaseAddress < (PVOID)((ULONG_PTR)MmNonPagedPoolStart + MmSizeOfNonPagedPoolInBytes))) || ((BaseAddress >= MmNonPagedPoolExpansionStart) && (BaseAddress < MmNonPagedPoolEnd))) { // // It did, so just use the pool to free this // ExFreePoolWithTag(BaseAddress, 'mCmM'); return; } /* Get the PTE and frame number for the allocation*/ PointerPte = MiAddressToPte(BaseAddress); PageFrameIndex = PFN_FROM_PTE(PointerPte); // // Now get the PFN entry for this, and make sure it's the correct one // Pfn1 = MiGetPfnEntry(PageFrameIndex); if ((!Pfn1) || (Pfn1->u3.e1.StartOfAllocation == 0)) { // // This probably means you did a free on an address that was in between // KeBugCheckEx(BAD_POOL_CALLER, 0x60, (ULONG_PTR)BaseAddress, 0, 0); } // // Now this PFN isn't the start of any allocation anymore, it's going out // StartPfn = Pfn1; Pfn1->u3.e1.StartOfAllocation = 0; /* Loop the PFNs until we find the one that marks the end of the allocation */ do { /* Make sure these are the pages we setup in the allocation routine */ ASSERT(Pfn1->u3.e2.ReferenceCount == 1); ASSERT(Pfn1->u2.ShareCount == 1); ASSERT(Pfn1->PteAddress == PointerPte); ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); ASSERT(Pfn1->u4.VerifierAllocation == 0); ASSERT(Pfn1->u3.e1.PrototypePte == 0); /* Set the special pending delete marker */ MI_SET_PFN_DELETED(Pfn1); /* Keep going for assertions */ PointerPte++; } while (Pfn1++->u3.e1.EndOfAllocation == 0); // // Found it, unmark it // Pfn1--; Pfn1->u3.e1.EndOfAllocation = 0; // // Now compute how many pages this represents // PageCount = (ULONG)(Pfn1 - StartPfn + 1); // // So we can know how much to unmap (recall we piggyback on I/O mappings) // MmUnmapIoSpace(BaseAddress, PageCount << PAGE_SHIFT); // // Lock the PFN database // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); // // Loop all the pages // LastPage = PageFrameIndex + PageCount; Pfn1 = MiGetPfnEntry(PageFrameIndex); do { /* Decrement the share count and move on */ MiDecrementShareCount(Pfn1++, PageFrameIndex++); } while (PageFrameIndex < LastPage); // // Release the PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); }
/* * @implemented */ VOID NTAPI MmProbeAndLockPages(IN PMDL Mdl, IN KPROCESSOR_MODE AccessMode, IN LOCK_OPERATION Operation) { PPFN_NUMBER MdlPages; PVOID Base, Address, LastAddress, StartAddress; ULONG LockPages, TotalPages; NTSTATUS Status = STATUS_SUCCESS; PEPROCESS CurrentProcess; NTSTATUS ProbeStatus; PMMPTE PointerPte, LastPte; PMMPDE PointerPde; #if (_MI_PAGING_LEVELS >= 3) PMMPDE PointerPpe; #endif #if (_MI_PAGING_LEVELS == 4) PMMPDE PointerPxe; #endif PFN_NUMBER PageFrameIndex; BOOLEAN UsePfnLock; KIRQL OldIrql; PMMPFN Pfn1; DPRINT("Probing MDL: %p\n", Mdl); // // Sanity checks // ASSERT(Mdl->ByteCount != 0); ASSERT(((ULONG)Mdl->ByteOffset & ~(PAGE_SIZE - 1)) == 0); ASSERT(((ULONG_PTR)Mdl->StartVa & (PAGE_SIZE - 1)) == 0); ASSERT((Mdl->MdlFlags & (MDL_PAGES_LOCKED | MDL_MAPPED_TO_SYSTEM_VA | MDL_SOURCE_IS_NONPAGED_POOL | MDL_PARTIAL | MDL_IO_SPACE)) == 0); // // Get page and base information // MdlPages = (PPFN_NUMBER)(Mdl + 1); Base = Mdl->StartVa; // // Get the addresses and how many pages we span (and need to lock) // Address = (PVOID)((ULONG_PTR)Base + Mdl->ByteOffset); LastAddress = (PVOID)((ULONG_PTR)Address + Mdl->ByteCount); LockPages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(Address, Mdl->ByteCount); ASSERT(LockPages != 0); /* Block invalid access */ if ((AccessMode != KernelMode) && ((LastAddress > (PVOID)MM_USER_PROBE_ADDRESS) || (Address >= LastAddress))) { /* Caller should be in SEH, raise the error */ *MdlPages = LIST_HEAD; ExRaiseStatus(STATUS_ACCESS_VIOLATION); } // // Get the process // if (Address <= MM_HIGHEST_USER_ADDRESS) { // // Get the process // CurrentProcess = PsGetCurrentProcess(); } else { // // No process // CurrentProcess = NULL; } // // Save the number of pages we'll have to lock, and the start address // TotalPages = LockPages; StartAddress = Address; /* Large pages not supported */ ASSERT(!MI_IS_PHYSICAL_ADDRESS(Address)); // // Now probe them // ProbeStatus = STATUS_SUCCESS; _SEH2_TRY { // // Enter probe loop // do { // // Assume failure // *MdlPages = LIST_HEAD; // // Read // *(volatile CHAR*)Address; // // Check if this is write access (only probe for user-mode) // if ((Operation != IoReadAccess) && (Address <= MM_HIGHEST_USER_ADDRESS)) { // // Probe for write too // ProbeForWriteChar(Address); } // // Next address... // Address = PAGE_ALIGN((ULONG_PTR)Address + PAGE_SIZE); // // Next page... // LockPages--; MdlPages++; } while (Address < LastAddress); // // Reset back to the original page // ASSERT(LockPages == 0); MdlPages = (PPFN_NUMBER)(Mdl + 1); } _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) { // // Oops :( // ProbeStatus = _SEH2_GetExceptionCode(); } _SEH2_END; // // So how did that go? // if (ProbeStatus != STATUS_SUCCESS) { // // Fail // DPRINT1("MDL PROBE FAILED!\n"); Mdl->Process = NULL; ExRaiseStatus(ProbeStatus); } // // Get the PTE and PDE // PointerPte = MiAddressToPte(StartAddress); PointerPde = MiAddressToPde(StartAddress); #if (_MI_PAGING_LEVELS >= 3) PointerPpe = MiAddressToPpe(StartAddress); #endif #if (_MI_PAGING_LEVELS == 4) PointerPxe = MiAddressToPxe(StartAddress); #endif // // Sanity check // ASSERT(MdlPages == (PPFN_NUMBER)(Mdl + 1)); // // Check what kind of operation this is // if (Operation != IoReadAccess) { // // Set the write flag // Mdl->MdlFlags |= MDL_WRITE_OPERATION; } else { // // Remove the write flag // Mdl->MdlFlags &= ~(MDL_WRITE_OPERATION); } // // Mark the MDL as locked *now* // Mdl->MdlFlags |= MDL_PAGES_LOCKED; // // Check if this came from kernel mode // if (Base > MM_HIGHEST_USER_ADDRESS) { // // We should not have a process // ASSERT(CurrentProcess == NULL); Mdl->Process = NULL; // // In kernel mode, we don't need to check for write access // Operation = IoReadAccess; // // Use the PFN lock // UsePfnLock = TRUE; OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } else { // // Sanity checks // ASSERT(TotalPages != 0); ASSERT(CurrentProcess == PsGetCurrentProcess()); // // Track locked pages // InterlockedExchangeAddSizeT(&CurrentProcess->NumberOfLockedPages, TotalPages); // // Save the process // Mdl->Process = CurrentProcess; /* Lock the process working set */ MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); UsePfnLock = FALSE; OldIrql = MM_NOIRQL; } // // Get the last PTE // LastPte = MiAddressToPte((PVOID)((ULONG_PTR)LastAddress - 1)); // // Loop the pages // do { // // Assume failure and check for non-mapped pages // *MdlPages = LIST_HEAD; while ( #if (_MI_PAGING_LEVELS == 4) (PointerPxe->u.Hard.Valid == 0) || #endif #if (_MI_PAGING_LEVELS >= 3) (PointerPpe->u.Hard.Valid == 0) || #endif (PointerPde->u.Hard.Valid == 0) || (PointerPte->u.Hard.Valid == 0)) { // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Access the page // Address = MiPteToAddress(PointerPte); //HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked Status = MmAccessFault(FALSE, Address, KernelMode, (PVOID)0xBADBADA3); if (!NT_SUCCESS(Status)) { // // Fail // DPRINT1("Access fault failed\n"); goto Cleanup; } // // What lock should we use? // if (UsePfnLock) { // // Grab the PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } else { /* Lock the process working set */ MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } } // // Check if this was a write or modify // if (Operation != IoReadAccess) { // // Check if the PTE is not writable // if (MI_IS_PAGE_WRITEABLE(PointerPte) == FALSE) { // // Check if it's copy on write // if (MI_IS_PAGE_COPY_ON_WRITE(PointerPte)) { // // Get the base address and allow a change for user-mode // Address = MiPteToAddress(PointerPte); if (Address <= MM_HIGHEST_USER_ADDRESS) { // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Access the page // //HACK: Pass a placeholder TrapInformation so the fault handler knows we're unlocked Status = MmAccessFault(TRUE, Address, KernelMode, (PVOID)0xBADBADA3); if (!NT_SUCCESS(Status)) { // // Fail // DPRINT1("Access fault failed\n"); goto Cleanup; } // // Re-acquire the lock // if (UsePfnLock) { // // Grab the PFN lock // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); } else { /* Lock the process working set */ MiLockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Start over // continue; } } // // Fail, since we won't allow this // Status = STATUS_ACCESS_VIOLATION; goto CleanupWithLock; } } // // Grab the PFN // PageFrameIndex = PFN_FROM_PTE(PointerPte); Pfn1 = MiGetPfnEntry(PageFrameIndex); if (Pfn1) { /* Either this is for kernel-mode, or the working set is held */ ASSERT((CurrentProcess == NULL) || (UsePfnLock == FALSE)); /* No Physical VADs supported yet */ if (CurrentProcess) ASSERT(CurrentProcess->PhysicalVadRoot == NULL); /* This address should already exist and be fully valid */ MiReferenceProbedPageAndBumpLockCount(Pfn1); } else { // // For I/O addresses, just remember this // Mdl->MdlFlags |= MDL_IO_SPACE; } // // Write the page and move on // *MdlPages++ = PageFrameIndex; PointerPte++; /* Check if we're on a PDE boundary */ if (MiIsPteOnPdeBoundary(PointerPte)) PointerPde++; #if (_MI_PAGING_LEVELS >= 3) if (MiIsPteOnPpeBoundary(PointerPte)) PointerPpe++; #endif #if (_MI_PAGING_LEVELS == 4) if (MiIsPteOnPxeBoundary(PointerPte)) PointerPxe++; #endif } while (PointerPte <= LastPte); // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } // // Sanity check // ASSERT((Mdl->MdlFlags & MDL_DESCRIBES_AWE) == 0); return; CleanupWithLock: // // This is the failure path // ASSERT(!NT_SUCCESS(Status)); // // What kind of lock were we using? // if (UsePfnLock) { // // Release PFN lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Release process working set */ MiUnlockProcessWorkingSet(CurrentProcess, PsGetCurrentThread()); } Cleanup: // // Pages must be locked so MmUnlock can work // ASSERT(Mdl->MdlFlags & MDL_PAGES_LOCKED); MmUnlockPages(Mdl); // // Raise the error // ExRaiseStatus(Status); }
static PMMPTE MiGetPteForProcess( PEPROCESS Process, PVOID Address, BOOLEAN Create) { MMPTE TmplPte, *Pte; /* Check if we need hypersapce mapping */ if (Address < MmSystemRangeStart && Process && Process != PsGetCurrentProcess()) { UNIMPLEMENTED; __debugbreak(); return NULL; } else if (Create) { KIRQL OldIrql; TmplPte.u.Long = 0; TmplPte.u.Flush.Valid = 1; TmplPte.u.Flush.Write = 1; /* All page table levels of user pages are user owned */ TmplPte.u.Flush.Owner = (Address < MmHighestUserAddress) ? 1 : 0; /* Lock the PFN database */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Get the PXE */ Pte = MiAddressToPxe(Address); if (!Pte->u.Hard.Valid) { TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(0); MI_WRITE_VALID_PTE(Pte, TmplPte); } /* Get the PPE */ Pte = MiAddressToPpe(Address); if (!Pte->u.Hard.Valid) { TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(1); MI_WRITE_VALID_PTE(Pte, TmplPte); } /* Get the PDE */ Pte = MiAddressToPde(Address); if (!Pte->u.Hard.Valid) { TmplPte.u.Hard.PageFrameNumber = MiRemoveZeroPage(2); MI_WRITE_VALID_PTE(Pte, TmplPte); } /* Unlock PFN database */ KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } else { /* Get the PXE */ Pte = MiAddressToPxe(Address); if (!Pte->u.Hard.Valid) return NULL; /* Get the PPE */ Pte = MiAddressToPpe(Address); if (!Pte->u.Hard.Valid) return NULL; /* Get the PDE */ Pte = MiAddressToPde(Address); if (!Pte->u.Hard.Valid) return NULL; } return MiAddressToPte(Address); }
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes) { PFN_NUMBER PageFrameNumber; PFN_COUNT SizeInPages, PageTableCount; ULONG i; KIRQL OldIrql; PLIST_ENTRY NextEntry, NextHead, LastHead; PMMPTE PointerPte, StartPte; PMMPDE PointerPde; ULONG EndAllocation; MMPTE TempPte; MMPDE TempPde; PMMPFN Pfn1; PVOID BaseVa, BaseVaStart; PMMFREE_POOL_ENTRY FreeEntry; PKSPIN_LOCK_QUEUE LockQueue; // // Figure out how big the allocation is in pages // SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes); // // Check for overflow // if (SizeInPages == 0) { // // Fail // return NULL; } // // Handle paged pool // if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Lock the paged pool mutex // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Find some empty allocation space // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, MmPagedPoolInfo.PagedPoolHint); if (i == 0xFFFFFFFF) { // // Get the page bit count // i = ((SizeInPages - 1) / PTE_COUNT) + 1; DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages); // // Check if there is enougn paged pool expansion space left // if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } // // Check if we'll have to expand past the last PTE we have available // if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // We can only support this much then // PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool); PageTableCount = (PFN_COUNT)(PointerPde + 1 - MmPagedPoolInfo.NextPdeForPagedPoolExpansion); ASSERT(PageTableCount < i); i = PageTableCount; } else { // // Otherwise, there is plenty of space left for this expansion // PageTableCount = i; } // // Get the template PDE we'll use to expand // TempPde = ValidKernelPde; // // Get the first PTE in expansion space // PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion; BaseVa = MiPdeToPte(PointerPde); BaseVaStart = BaseVa; // // Lock the PFN database and loop pages // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // It should not already be valid // ASSERT(PointerPde->u.Hard.Valid == 0); /* Request a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPde.u.Hard.PageFrameNumber = PageFrameNumber; #if (_MI_PAGING_LEVELS >= 3) /* On PAE/x64 systems, there's no double-buffering */ ASSERT(FALSE); #else // // Save it into our double-buffered system page directory // MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde; /* Initialize the PFN */ MiInitializePfnForOtherProcess(PageFrameNumber, (PMMPTE)PointerPde, MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]); /* Write the actual PDE now */ // MI_WRITE_VALID_PDE(PointerPde, TempPde); #endif // // Move on to the next expansion address // PointerPde++; BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE); i--; } while (i > 0); // // Release the PFN database lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // These pages are now available, clear their availablity bits // EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion - (PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) * PTE_COUNT; RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, EndAllocation, PageTableCount * PTE_COUNT); // // Update the next expansion location // MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount; // // Zero out the newly available memory // RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE); // // Now try consuming the pages again // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, 0); if (i == 0xFFFFFFFF) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } } // // Update the pool hint if the request was just one page // if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1; // // Update the end bitmap so we know the bounds of this allocation when // the time comes to free it // EndAllocation = i + SizeInPages - 1; RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation); // // Now we can release the lock (it mainly protects the bitmap) // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // Now figure out where this allocation starts // BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT)); // // Flush the TLB // KeFlushEntireTb(TRUE, TRUE); /* Setup a demand-zero writable PTE */ MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE); // // Find the first and last PTE, then loop them all // PointerPte = MiAddressToPte(BaseVa); StartPte = PointerPte + SizeInPages; do { // // Write the demand zero PTE and keep going // MI_WRITE_INVALID_PTE(PointerPte, TempPte); } while (++PointerPte < StartPte); // // Return the allocation address to the caller // return BaseVa; } // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Allocations of less than 4 pages go into their individual buckets // i = SizeInPages - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; // // Loop through all the free page lists based on the page index // NextHead = &MmNonPagedPoolFreeListHead[i]; LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS]; // // Acquire the nonpaged pool lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); do { // // Now loop through all the free page entries in this given list // NextEntry = NextHead->Flink; while (NextEntry != NextHead) { /* Is freed non paged pool enabled */ if (MmProtectFreedNonPagedPool) { /* We need to be able to touch this page, unprotect it */ MiUnProtectFreeNonPagedPool(NextEntry, 0); } // // Grab the entry and see if it can handle our allocation // FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List); ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE); if (FreeEntry->Size >= SizeInPages) { // // It does, so consume the pages from here // FreeEntry->Size -= SizeInPages; // // The allocation will begin in this free page area // BaseVa = (PVOID)((ULONG_PTR)FreeEntry + (FreeEntry->Size << PAGE_SHIFT)); /* Remove the item from the list, depending if pool is protected */ if (MmProtectFreedNonPagedPool) MiProtectedPoolRemoveEntryList(&FreeEntry->List); else RemoveEntryList(&FreeEntry->List); // // However, check if its' still got space left // if (FreeEntry->Size != 0) { /* Check which list to insert this entry into */ i = FreeEntry->Size - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; /* Insert the entry into the free list head, check for prot. pool */ if (MmProtectFreedNonPagedPool) MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE); else InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List); /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } // // Grab the PTE for this allocation // PointerPte = MiAddressToPte(BaseVa); ASSERT(PointerPte->u.Hard.Valid == 1); // // Grab the PFN NextEntry and index // Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); // // Now mark it as the beginning of an allocation // ASSERT(Pfn1->u3.e1.StartOfAllocation == 0); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as special pool if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) { Pfn1->u4.VerifierAllocation = 1; } // // Check if the allocation is larger than one page // if (SizeInPages != 1) { // // Navigate to the last PFN entry and PTE // PointerPte += SizeInPages - 1; ASSERT(PointerPte->u.Hard.Valid == 1); Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber); } // // Mark this PFN as the last (might be the same as the first) // ASSERT(Pfn1->u3.e1.EndOfAllocation == 0); Pfn1->u3.e1.EndOfAllocation = 1; // // Release the nonpaged pool lock, and return the allocation // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); return BaseVa; } // // Try the next free page entry // NextEntry = FreeEntry->List.Flink; /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } } while (++NextHead < LastHead); // // If we got here, we're out of space. // Start by releasing the lock // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Allocate some system PTEs // StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion); PointerPte = StartPte; if (StartPte == NULL) { // // Ran out of memory // DPRINT1("Out of NP Expansion Pool\n"); return NULL; } // // Acquire the pool lock now // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); // // Lock the PFN database too // LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock]; KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); // // Loop the pages // TempPte = ValidKernelPte; do { /* Allocate a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->PteAddress = PointerPte; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u4.VerifierAllocation = 0; /* Write the PTE for it */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MI_WRITE_VALID_PTE(PointerPte++, TempPte); } while (--SizeInPages > 0); // // This is the last page // Pfn1->u3.e1.EndOfAllocation = 1; // // Get the first page and mark it as such // Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as a verifier allocation if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1; // // Release the PFN and nonpaged pool lock // KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Return the address // return MiPteToAddress(StartPte); }
PVOID NTAPI MmAllocateSpecialPool(SIZE_T NumberOfBytes, ULONG Tag, POOL_TYPE PoolType, ULONG SpecialType) { KIRQL Irql; MMPTE TempPte = ValidKernelPte; PMMPTE PointerPte; PFN_NUMBER PageFrameNumber; LARGE_INTEGER TickCount; PVOID Entry; PPOOL_HEADER Header; DPRINT1("MmAllocateSpecialPool(%x %x %x %x)\n", NumberOfBytes, Tag, PoolType, SpecialType); /* Check if the pool is initialized and quit if it's not */ if (!MiSpecialPoolFirstPte) return NULL; /* Get the pool type */ PoolType &= BASE_POOL_TYPE_MASK; /* Check whether current IRQL matches the pool type */ Irql = KeGetCurrentIrql(); if (((PoolType == PagedPool) && (Irql > APC_LEVEL)) || ((PoolType != PagedPool) && (Irql > DISPATCH_LEVEL))) { /* Bad caller */ KeBugCheckEx(SPECIAL_POOL_DETECTED_MEMORY_CORRUPTION, Irql, PoolType, NumberOfBytes, 0x30); } /* TODO: Take into account various limitations */ /*if ((PoolType != NonPagedPool) && MiSpecialPagesNonPaged > MiSpecialPagesNonPagedMaximum)*/ /* Lock PFN database */ Irql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Reject allocation in case amount of available pages is too small */ if (MmAvailablePages < 0x100) { /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); DPRINT1("Special pool: MmAvailablePages 0x%x is too small\n", MmAvailablePages); return NULL; } /* Reject allocation if special pool PTE list is exhausted */ if (MiSpecialPoolFirstPte->u.List.NextEntry == MM_EMPTY_PTE_LIST) { /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); DPRINT1("Special pool: No PTEs left!\n"); /* TODO: Expand the special pool */ return NULL; } /* Save allocation time */ KeQueryTickCount(&TickCount); /* Get a pointer to the first PTE */ PointerPte = MiSpecialPoolFirstPte; /* Set the first PTE pointer to the next one in the list */ MiSpecialPoolFirstPte = MmSystemPteBase + PointerPte->u.List.NextEntry; /* Allocate a physical page */ PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Initialize PFN and make it valid */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MiInitializePfnAndMakePteValid(PageFrameNumber, PointerPte, TempPte); /* Release the PFN database lock */ KeReleaseQueuedSpinLock(LockQueuePfnLock, Irql); /* Put some content into the page. Low value of tick count would do */ Entry = MiPteToAddress(PointerPte); RtlFillMemory(Entry, PAGE_SIZE, TickCount.LowPart); /* Calculate header and entry addresses */ if ((SpecialType != 0) && ((SpecialType == 1) || (!MmSpecialPoolCatchOverruns))) { /* We catch underruns. Data is at the beginning of the page */ Header = (PPOOL_HEADER)((PUCHAR)Entry + PAGE_SIZE - sizeof(POOL_HEADER)); } else { /* We catch overruns. Data is at the end of the page */ Header = (PPOOL_HEADER)Entry; Entry = (PVOID)((ULONG_PTR)((PUCHAR)Entry - NumberOfBytes + PAGE_SIZE) & ~((LONG_PTR)sizeof(POOL_HEADER) - 1)); } /* Initialize the header */ RtlZeroMemory(Header, sizeof(POOL_HEADER)); /* Save allocation size there */ Header->Ulong1 = (ULONG)NumberOfBytes; /* Make sure it's all good */ ASSERT((NumberOfBytes <= PAGE_SIZE - sizeof(POOL_HEADER)) && (PAGE_SIZE <= 32 * 1024)); /* Mark it as paged or nonpaged */ if (PoolType == PagedPool) { /* Add pagedpool flag into the pool header too */ Header->Ulong1 |= SPECIAL_POOL_PAGED; /* Also mark the next PTE as special-pool-paged */ PointerPte[1].u.Soft.PageFileHigh |= SPECIAL_POOL_PAGED_PTE; } else { /* Mark the next PTE as special-pool-nonpaged */ PointerPte[1].u.Soft.PageFileHigh |= SPECIAL_POOL_NONPAGED_PTE; } /* Finally save tag and put allocation time into the header's blocksize. That time will be used to check memory consistency within the allocated page. */ Header->PoolTag = Tag; Header->BlockSize = (USHORT)TickCount.LowPart; DPRINT1("%p\n", Entry); return Entry; }