CConnectionPool::~CConnectionPool()
{
PSLIST_ENTRY entry;
if (this->list)
{
while (NULL != (entry = InterlockedPopEntrySList(this->list)))
{
CloseHandle(((PCONNECTION_ENTRY)entry)->connection);
_aligned_free(entry);
}
_aligned_free(this->list);
this->list = NULL;
}
}
static DWORD WINAPI printer_thread_func(LPVOID arg)
{
IRP* irp;
PRINTER_DEVICE* printer_dev = (PRINTER_DEVICE*) arg;
HANDLE obj[] = {printer_dev->event, printer_dev->stopEvent};
UINT error = CHANNEL_RC_OK;
while (1)
{
DWORD rc = WaitForMultipleObjects(2, obj, FALSE, INFINITE);
if (rc == WAIT_FAILED)
{
error = GetLastError();
WLog_ERR(TAG, "WaitForMultipleObjects failed with error %"PRIu32"!", error);
break;
}
if (rc == WAIT_OBJECT_0 + 1)
break;
else if (rc != WAIT_OBJECT_0)
continue;
ResetEvent(printer_dev->event);
irp = (IRP*) InterlockedPopEntrySList(printer_dev->pIrpList);
if (irp == NULL)
{
WLog_ERR(TAG, "InterlockedPopEntrySList failed!");
error = ERROR_INTERNAL_ERROR;
break;
}
if ((error = printer_process_irp(printer_dev, irp)))
{
WLog_ERR(TAG, "printer_process_irp failed with error %"PRIu32"!", error);
break;
}
}
if (error && printer_dev->rdpcontext)
setChannelError(printer_dev->rdpcontext, error,
"printer_thread_func reported an error");
ExitThread(error);
return error;
}
static void parallel_process_irp_list(PARALLEL_DEVICE* parallel)
{
IRP* irp;
while (1)
{
if (freerdp_thread_is_stopped(parallel->thread))
break;
irp = (IRP*) InterlockedPopEntrySList(parallel->pIrpList);
if (irp == NULL)
break;
parallel_process_irp(parallel, irp);
}
}
static void parallel_free(DEVICE* device)
{
IRP* irp;
PARALLEL_DEVICE* parallel = (PARALLEL_DEVICE*) device;
DEBUG_SVC("freeing device");
freerdp_thread_stop(parallel->thread);
freerdp_thread_free(parallel->thread);
while ((irp = (IRP*) InterlockedPopEntrySList(parallel->pIrpList)) != NULL)
irp->Discard(irp);
_aligned_free(parallel->pIrpList);
free(parallel);
}
static void smartcard_process_irp_list(SMARTCARD_DEVICE* smartcard)
{
IRP* irp;
while (1)
{
if (WaitForSingleObject(smartcard->stopEvent, 0) == WAIT_OBJECT_0)
break;
irp = (IRP*) InterlockedPopEntrySList(smartcard->pIrpList);
if (irp == NULL)
break;
smartcard_process_irp(smartcard, irp);
}
}
static void drive_process_irp_list(DRIVE_DEVICE* disk)
{
IRP* irp;
while (1)
{
if (WaitForSingleObject(disk->stopEvent, 0) == WAIT_OBJECT_0)
break;
irp = (IRP*) InterlockedPopEntrySList(disk->pIrpList);
if (irp == NULL)
break;
drive_process_irp(disk, irp);
}
}
// @return OVERLAPPED initialized for I/O starting at offset,
// or 0 if all available structures have already been allocated.
OVERLAPPED* Allocate(off_t offset)
{
Entry* entry = (Entry*)InterlockedPopEntrySList(&freelist);
if(!entry)
return 0;
OVERLAPPED& ovl = entry->ovl;
ovl.Internal = 0;
ovl.InternalHigh = 0;
ovl.Offset = u64_lo(offset);
ovl.OffsetHigh = u64_hi(offset);
ovl.hEvent = 0; // (notification is via IOCP and/or polling)
cpu_AtomicAdd(&extant, +1);
return &ovl;
}
static void printer_process_irp_list(PRINTER_DEVICE* printer_dev)
{
IRP* irp;
while (1)
{
if (freerdp_thread_is_stopped(printer_dev->thread))
break;
irp = (IRP*) InterlockedPopEntrySList(printer_dev->pIrpList);
if (irp == NULL)
break;
printer_process_irp(printer_dev, irp);
}
}
static void printer_free(DEVICE* device)
{
IRP* irp;
PRINTER_DEVICE* printer_dev = (PRINTER_DEVICE*) device;
freerdp_thread_stop(printer_dev->thread);
freerdp_thread_free(printer_dev->thread);
while ((irp = (IRP*) InterlockedPopEntrySList(printer_dev->pIrpList)) != NULL)
irp->Discard(irp);
_aligned_free(printer_dev->pIrpList);
if (printer_dev->printer)
printer_dev->printer->Free(printer_dev->printer);
free(printer_dev->device.name);
free(printer_dev);
}
void HttpInputQueueDrain(PHTTP_LISTENER listener)
{
PHTTP_IO_CONTEXT context;
PSLIST_ENTRY entry;
PLOOKASIDE inputQueue = listener->HttpInputQueue;
while(true)
{
entry = InterlockedPopEntrySList(&inputQueue->Header);
if(entry != NULL)
{
context = CONTAINING_RECORD(entry, HTTP_IO_CONTEXT, LookAsideEntry);
HttpListenerCompleteIo(context);
}
else
{
break;
}
}
}
PTRANSFER_PACKET DequeueFreeTransferPacket(PDEVICE_OBJECT Fdo, BOOLEAN AllocIfNeeded)
{
PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
PTRANSFER_PACKET pkt;
PSLIST_ENTRY slistEntry;
slistEntry = InterlockedPopEntrySList(&fdoData->FreeTransferPacketsList);
if (slistEntry){
slistEntry->Next = NULL;
pkt = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
InterlockedDecrement(&fdoData->NumFreeTransferPackets);
// when dequeue'ing the packet, also reset the history data
HISTORYINITIALIZERETRYLOGS(pkt);
}
else {
if (AllocIfNeeded){
/*
* We are in stress and have run out of lookaside packets.
* In order to service the current transfer,
* allocate an extra packet.
* We will free it lazily when we are out of stress.
*/
pkt = NewTransferPacket(Fdo);
if (pkt){
InterlockedIncrement(&fdoData->NumTotalTransferPackets);
fdoData->DbgPeakNumTransferPackets = max(fdoData->DbgPeakNumTransferPackets, fdoData->NumTotalTransferPackets);
}
else {
TracePrint((TRACE_LEVEL_WARNING, TRACE_FLAG_RW, "DequeueFreeTransferPacket: packet allocation failed"));
}
}
else {
pkt = NULL;
}
}
return pkt;
}
PHTTP_IO_CONTEXT GetIOContext()
{
PHTTP_IO_CONTEXT pContext;
PLOOKASIDE pCacheEntry;
PSLIST_ENTRY pEntry;
pCacheEntry = &IoContextCacheList[IO_CONTEXT_PROC_INDEX];
pEntry = InterlockedPopEntrySList(&pCacheEntry->Header);
if(pEntry != NULL)
{
// Return address of the containing structure.
pContext = CONTAINING_RECORD(pEntry, HTTP_IO_CONTEXT, LookAsideEntry);
}
else
{
pContext = (PHTTP_IO_CONTEXT)ALLOC_MEM(sizeof(HTTP_IO_CONTEXT));
}
ZeroMemory(pContext, sizeof(HTTP_IO_CONTEXT));
return pContext;
}
PTRANSFER_PACKET DequeueFreeTransferPacket(PDEVICE_OBJECT Fdo, BOOLEAN AllocIfNeeded)
{
PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
PTRANSFER_PACKET pkt;
PSINGLE_LIST_ENTRY slistEntry;
KIRQL oldIrql;
slistEntry = InterlockedPopEntrySList(&fdoData->FreeTransferPacketsList);
if (slistEntry){
slistEntry->Next = NULL;
pkt = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
ASSERT(fdoData->NumFreeTransferPackets > 0);
InterlockedDecrement(&fdoData->NumFreeTransferPackets);
}
else {
if (AllocIfNeeded){
/*
* We are in stress and have run out of lookaside packets.
* In order to service the current transfer,
* allocate an extra packet.
* We will free it lazily when we are out of stress.
*/
pkt = NewTransferPacket(Fdo);
if (pkt){
InterlockedIncrement(&fdoData->NumTotalTransferPackets);
fdoData->DbgPeakNumTransferPackets = max(fdoData->DbgPeakNumTransferPackets, fdoData->NumTotalTransferPackets);
}
else {
DBGWARN(("DequeueFreeTransferPacket: packet allocation failed"));
}
}
else {
pkt = NULL;
}
}
return pkt;
}
static void drive_free(DEVICE* device)
{
IRP* irp;
DRIVE_FILE* file;
DRIVE_DEVICE* disk = (DRIVE_DEVICE*) device;
SetEvent(disk->stopEvent);
CloseHandle(disk->thread);
CloseHandle(disk->irpEvent);
while ((irp = (IRP*) InterlockedPopEntrySList(disk->pIrpList)) != NULL)
irp->Discard(irp);
_aligned_free(disk->pIrpList);
while ((file = (DRIVE_FILE*) list_dequeue(disk->files)) != NULL)
drive_file_free(file);
list_free(disk->files);
free(disk);
}
static void cp_worker_thread(void *param)
{
SLIST_ENTRY *entry;
req_part *part;
req_item *item;
const char *in;
char *out;
u64 offset;
u32 length;
do
{
KeWaitForSingleObject(&pool_signal_event, Executive, KernelMode, FALSE, NULL);
KeClearEvent(&pool_signal_event);
while (entry = InterlockedPopEntrySList(&pool_head))
{
part = CONTAINING_RECORD(entry, req_part, entry);
item = part->item;
in = item->in + part->offset;
out = item->out + part->offset;
offset = item->offset + part->offset;
length = part->length;
if (item->is_encrypt != 0) {
xts_encrypt(in, out, length, offset, item->key);
} else {
xts_decrypt(in, out, length, offset, item->key);
}
if (lock_xchg_add(&item->length, 0-length) == length) {
item->on_complete(item->param);
ExFreeToNPagedLookasideList(&pool_req_mem, item);
}
}
} while (pool_enabled != 0);
PsTerminateSystemThread(STATUS_SUCCESS);
}
void
cubeb_destroy(cubeb * ctx)
{
DWORD rv;
assert(!InterlockedPopEntrySList(ctx->work));
if (ctx->thread) {
ctx->shutdown = 1;
SetEvent(ctx->event);
rv = WaitForSingleObject(ctx->thread, INFINITE);
assert(rv == WAIT_OBJECT_0);
CloseHandle(ctx->thread);
}
if (ctx->event) {
CloseHandle(ctx->event);
}
_aligned_free(ctx->work);
free(ctx);
}
int TestInterlockedSList(int argc, char* argv[])
{
ULONG Count;
WINPR_PSLIST_ENTRY pFirstEntry;
WINPR_PSLIST_ENTRY pListEntry;
WINPR_PSLIST_HEADER pListHead;
PPROGRAM_ITEM pProgramItem;
/* Initialize the list header to a MEMORY_ALLOCATION_ALIGNMENT boundary. */
pListHead = (WINPR_PSLIST_HEADER) _aligned_malloc(sizeof(WINPR_SLIST_HEADER), MEMORY_ALLOCATION_ALIGNMENT);
if (!pListHead)
{
printf("Memory allocation failed.\n");
return -1;
}
InitializeSListHead(pListHead);
/* Insert 10 items into the list. */
for (Count = 1; Count <= 10; Count += 1)
{
pProgramItem = (PPROGRAM_ITEM) _aligned_malloc(sizeof(PROGRAM_ITEM), MEMORY_ALLOCATION_ALIGNMENT);
if (!pProgramItem)
{
printf("Memory allocation failed.\n");
return -1;
}
pProgramItem->Signature = Count;
pFirstEntry = InterlockedPushEntrySList(pListHead, &(pProgramItem->ItemEntry));
}
/* Remove 10 items from the list and display the signature. */
for (Count = 10; Count >= 1; Count -= 1)
{
pListEntry = InterlockedPopEntrySList(pListHead);
if (!pListEntry)
{
printf("List is empty.\n");
return -1;
}
pProgramItem = (PPROGRAM_ITEM) pListEntry;
printf("Signature is %d\n", (int) pProgramItem->Signature);
/*
* This example assumes that the SLIST_ENTRY structure is the
* first member of the structure. If your structure does not
* follow this convention, you must compute the starting address
* of the structure before calling the free function.
*/
_aligned_free(pListEntry);
}
/* Flush the list and verify that the items are gone. */
pListEntry = InterlockedFlushSList(pListHead);
pFirstEntry = InterlockedPopEntrySList(pListHead);
if (pFirstEntry)
{
printf("Error: List is not empty.\n");
return -1;
}
_aligned_free(pListHead);
return 0;
}
short TamperNoServer::process(DivertPacket *& dPacket)
{
/* Check the server rules */
switch ((*ppNoServerConfig)->noServersType)
{
case NO_SERVERS_CUSTOM:
{
while (true)
{
/* Pull an entry off of the list */
TamperNoServerEntry* listEntry = reinterpret_cast<TamperNoServerEntry*>(InterlockedPopEntrySList((*ppNoServerConfig)->noServersList));
/* If it is null, we've reached the end of the list and should leave the loop */
if (listEntry == nullptr)
{
break;
}
/* If our version doesn't match the entry version, we have old data and need to clear our list */
if (serverListVersion != listEntry->version)
{
serverList.clear();
serverListVersion = listEntry->version;
}
/* Add the port to the list and then free the list entry memory */
serverList.push_back(reinterpret_cast<TamperNoServerEntry*>(listEntry)->server);
_aligned_free(listEntry);
listEntry = nullptr;
/* Loop continues until we get a null pop entry */
}
break;
}
case NO_SERVERS_OFF:
default:
return 0;
}
/* Drop any packets traveling over blocked ports */
WINDIVERT_IPHDR *iphdr = nullptr;
WINDIVERT_TCPHDR *tcphdr = nullptr;
WINDIVERT_UDPHDR *udphdr = nullptr;
WinDivertHelperParsePacket(dPacket->packet, dPacket->packetLength, &iphdr, 0, 0, 0, &tcphdr, &udphdr, 0, 0);
if (iphdr)
{
for (UINT32 server : serverList)
{
if (iphdr->DstAddr == server || iphdr->SrcAddr == server)
{
/* Drop packet */
free(dPacket);
dPacket = nullptr;
break;
}
}
return 0;
}
/* No IP header. Skip this packet */
return 0;
}
/*
* @implemented
*/
NTSTATUS
NTAPI
IoSetIoCompletion(IN PVOID IoCompletion,
IN PVOID KeyContext,
IN PVOID ApcContext,
IN NTSTATUS IoStatus,
IN ULONG_PTR IoStatusInformation,
IN BOOLEAN Quota)
{
PKQUEUE Queue = (PKQUEUE)IoCompletion;
PNPAGED_LOOKASIDE_LIST List;
PKPRCB Prcb = KeGetCurrentPrcb();
PIOP_MINI_COMPLETION_PACKET Packet;
/* Get the P List */
List = (PNPAGED_LOOKASIDE_LIST)Prcb->
PPLookasideList[LookasideCompletionList].P;
/* Try to allocate the Packet */
List->L.TotalAllocates++;
Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead);
/* Check if that failed, use the L list if it did */
if (!Packet)
{
/* Let the balancer know */
List->L.AllocateMisses++;
/* Get L List */
List = (PNPAGED_LOOKASIDE_LIST)Prcb->
PPLookasideList[LookasideCompletionList].L;
/* Try to allocate the Packet */
List->L.TotalAllocates++;
Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead);
}
/* Still failed, use pool */
if (!Packet)
{
/* Let the balancer know */
List->L.AllocateMisses++;
/* Allocate from Nonpaged Pool */
Packet = ExAllocatePoolWithTag(NonPagedPool, sizeof(*Packet), IOC_TAG);
}
/* Make sure we have one by now... */
if (Packet)
{
/* Set up the Packet */
Packet->PacketType = IopCompletionPacketMini;
Packet->KeyContext = KeyContext;
Packet->ApcContext = ApcContext;
Packet->IoStatus = IoStatus;
Packet->IoStatusInformation = IoStatusInformation;
/* Insert the Queue */
KeInsertQueue(Queue, &Packet->ListEntry);
}
else
{
/* Out of memory, fail */
return STATUS_INSUFFICIENT_RESOURCES;
}
/* Return Success */
return STATUS_SUCCESS;
}
PVOID
ExInitializeNBQueueHead (
IN PSLIST_HEADER SlistHead
)
/*++
Routine Description:
This function initializes a non-blocking queue header.
N.B. It is assumed that the specified SLIST has been populated with
non-blocking queue nodes prior to calling this routine.
Arguments:
SlistHead - Supplies a pointer to an SLIST header.
Return Value:
If the non-blocking queue is successfully initialized, then the
address of the queue header is returned as the function value.
Otherwise, NULL is returned as the function value.
--*/
{
PNBQUEUE_HEADER QueueHead;
PNBQUEUE_NODE QueueNode;
//
// Attempt to allocate the queue header. If the allocation fails, then
// return NULL.
//
#if defined(_KERNEL_MODE)
QueueHead = (PNBQUEUE_HEADER)ExAllocatePoolWithTag(NonPagedPool,
sizeof(NBQUEUE_HEADER),
'hqBN');
#else
QueueHead = (PNBQUEUE_HEADER)HeapAlloc(GetProcessHeap(), 0,
sizeof(NBQUEUE_HEADER));
#endif
if (QueueHead == NULL) {
return NULL;
}
//
// Attempt to allocate a queue node from the specified SLIST. If a node
// can be allocated, then initialize the non-blocking queue header and
// return the address of the queue header. Otherwise, free the queue
// header and return NULL.
//
QueueHead->SlistHead = SlistHead;
QueueNode = (PNBQUEUE_NODE)InterlockedPopEntrySList(QueueHead->SlistHead);
if (QueueNode != NULL) {
//
// Initialize the queue node next pointer and value.
//
QueueNode->Next.Data = 0;
QueueNode->Value = 0;
//
// Initialize the head and tail pointers in the queue header.
//
PackNBQPointer(&QueueHead->Head, QueueNode);
QueueHead->Head.Count = 0;
PackNBQPointer(&QueueHead->Tail, QueueNode);
QueueHead->Tail.Count = 0;
return QueueHead;
} else {
#if defined(_KERNEL_MODE)
ExFreePool(QueueHead);
#else
HeapFree(GetProcessHeap(), 0, QueueHead);
#endif
return NULL;
}
}
SListEntry* SListImpl::pop()
{
return reinterpret_cast<SListEntry*>(InterlockedPopEntrySList(getDetail(this)));
}
BOOLEAN
ExInsertTailNBQueue (
IN PVOID Header,
IN ULONG64 Value
)
/*++
Routine Description:
This function inserts the specific data value at the tail of the
specified non-blocking queue.
Arguments:
Header - Supplies an opaque pointer to a non-blocking queue header.
Value - Supplies a pointer to an opaque data value.
Return Value:
If the specified opaque data value is successfully inserted at the tail
of the specified non-blocking queue, then a value of TRUE is returned as
the function value. Otherwise, a value of FALSE is returned.
N.B. FALSE is returned if a queue node cannot be allocated from the
associated SLIST.
--*/
{
NBQUEUE_POINTER Insert;
NBQUEUE_POINTER Next;
PNBQUEUE_NODE NextNode;
PNBQUEUE_HEADER QueueHead;
PNBQUEUE_NODE QueueNode;
NBQUEUE_POINTER Tail;
PNBQUEUE_NODE TailNode;
//
// Attempt to allocate a queue node from the SLIST associated with
// the specified non-blocking queue. If a node can be allocated, then
// the node is inserted at the tail of the specified non-blocking
// queue, and TRUE is returned as the function value. Otherwise, FALSE
// is returned.
//
QueueHead = (PNBQUEUE_HEADER)Header;
QueueNode = (PNBQUEUE_NODE)InterlockedPopEntrySList(QueueHead->SlistHead);
if (QueueNode != NULL) {
//
// Initialize the queue node next pointer and value.
//
QueueNode->Next.Data = 0;
QueueNode->Value = Value;
//
// The following loop is executed until the specified entry can
// be safely inserted at the tail of the specified non-blocking
// queue.
//
do {
//
// Read the tail queue pointer and the next queue pointer of
// the tail queue pointer making sure the two pointers are
// coherent.
//
Tail.Data = ReadForWriteAccess((volatile LONG64 *)(&QueueHead->Tail.Data));
TailNode = UnpackNBQPointer(&Tail);
Next.Data = *((volatile LONG64 *)(&TailNode->Next.Data));
QueueNode->Next.Count = Tail.Count + 1;
if (Tail.Data == *((volatile LONG64 *)(&QueueHead->Tail.Data))) {
//
// If the tail is pointing to the last node in the list,
// then attempt to insert the new node at the end of the
// list. Otherwise, the tail is not pointing to the last
// node in the list and an attempt is made to move the
// tail pointer to the next node.
//
NextNode = UnpackNBQPointer(&Next);
if (NextNode == NULL) {
PackNBQPointer(&Insert, QueueNode);
Insert.Count = Next.Count + 1;
if (InterlockedCompareExchange64(&TailNode->Next.Data,
Insert.Data,
Next.Data) == Next.Data) {
break;
}
} else {
PackNBQPointer(&Insert, NextNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
}
}
} while (TRUE);
//
// Attempt to move the tail to the new tail node.
//
PackNBQPointer(&Insert, QueueNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
return TRUE;
} else {
return FALSE;
}
}
PVOID
NTAPI
MiAllocatePoolPages(IN POOL_TYPE PoolType,
IN SIZE_T SizeInBytes)
{
PFN_NUMBER PageFrameNumber;
PFN_COUNT SizeInPages, PageTableCount;
ULONG i;
KIRQL OldIrql;
PLIST_ENTRY NextEntry, NextHead, LastHead;
PMMPTE PointerPte, StartPte;
PMMPDE PointerPde;
ULONG EndAllocation;
MMPTE TempPte;
MMPDE TempPde;
PMMPFN Pfn1;
PVOID BaseVa, BaseVaStart;
PMMFREE_POOL_ENTRY FreeEntry;
PKSPIN_LOCK_QUEUE LockQueue;
//
// Figure out how big the allocation is in pages
//
SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes);
//
// Check for overflow
//
if (SizeInPages == 0)
{
//
// Fail
//
return NULL;
}
//
// Handle paged pool
//
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
{
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Lock the paged pool mutex
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Find some empty allocation space
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
MmPagedPoolInfo.PagedPoolHint);
if (i == 0xFFFFFFFF)
{
//
// Get the page bit count
//
i = ((SizeInPages - 1) / PTE_COUNT) + 1;
DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages);
//
// Check if there is enougn paged pool expansion space left
//
if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
//
// Check if we'll have to expand past the last PTE we have available
//
if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// We can only support this much then
//
PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool);
PageTableCount = (PFN_COUNT)(PointerPde + 1 -
MmPagedPoolInfo.NextPdeForPagedPoolExpansion);
ASSERT(PageTableCount < i);
i = PageTableCount;
}
else
{
//
// Otherwise, there is plenty of space left for this expansion
//
PageTableCount = i;
}
//
// Get the template PDE we'll use to expand
//
TempPde = ValidKernelPde;
//
// Get the first PTE in expansion space
//
PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
BaseVa = MiPdeToPte(PointerPde);
BaseVaStart = BaseVa;
//
// Lock the PFN database and loop pages
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// It should not already be valid
//
ASSERT(PointerPde->u.Hard.Valid == 0);
/* Request a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
/* On PAE/x64 systems, there's no double-buffering */
ASSERT(FALSE);
#else
//
// Save it into our double-buffered system page directory
//
MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;
/* Initialize the PFN */
MiInitializePfnForOtherProcess(PageFrameNumber,
(PMMPTE)PointerPde,
MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]);
/* Write the actual PDE now */
// MI_WRITE_VALID_PDE(PointerPde, TempPde);
#endif
//
// Move on to the next expansion address
//
PointerPde++;
BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
i--;
} while (i > 0);
//
// Release the PFN database lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// These pages are now available, clear their availablity bits
//
EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
PTE_COUNT;
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
EndAllocation,
PageTableCount * PTE_COUNT);
//
// Update the next expansion location
//
MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount;
//
// Zero out the newly available memory
//
RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE);
//
// Now try consuming the pages again
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
0);
if (i == 0xFFFFFFFF)
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
}
//
// Update the pool hint if the request was just one page
//
if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;
//
// Update the end bitmap so we know the bounds of this allocation when
// the time comes to free it
//
EndAllocation = i + SizeInPages - 1;
RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);
//
// Now we can release the lock (it mainly protects the bitmap)
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// Now figure out where this allocation starts
//
BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));
//
// Flush the TLB
//
KeFlushEntireTb(TRUE, TRUE);
/* Setup a demand-zero writable PTE */
MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
//
// Find the first and last PTE, then loop them all
//
PointerPte = MiAddressToPte(BaseVa);
StartPte = PointerPte + SizeInPages;
do
{
//
// Write the demand zero PTE and keep going
//
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
} while (++PointerPte < StartPte);
//
// Return the allocation address to the caller
//
return BaseVa;
}
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Allocations of less than 4 pages go into their individual buckets
//
i = SizeInPages - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
//
// Loop through all the free page lists based on the page index
//
NextHead = &MmNonPagedPoolFreeListHead[i];
LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
//
// Acquire the nonpaged pool lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
do
{
//
// Now loop through all the free page entries in this given list
//
NextEntry = NextHead->Flink;
while (NextEntry != NextHead)
{
/* Is freed non paged pool enabled */
if (MmProtectFreedNonPagedPool)
{
/* We need to be able to touch this page, unprotect it */
MiUnProtectFreeNonPagedPool(NextEntry, 0);
}
//
// Grab the entry and see if it can handle our allocation
//
FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
if (FreeEntry->Size >= SizeInPages)
{
//
// It does, so consume the pages from here
//
FreeEntry->Size -= SizeInPages;
//
// The allocation will begin in this free page area
//
BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
(FreeEntry->Size << PAGE_SHIFT));
/* Remove the item from the list, depending if pool is protected */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolRemoveEntryList(&FreeEntry->List);
else
RemoveEntryList(&FreeEntry->List);
//
// However, check if its' still got space left
//
if (FreeEntry->Size != 0)
{
/* Check which list to insert this entry into */
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
/* Insert the entry into the free list head, check for prot. pool */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
else
InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
//
// Grab the PTE for this allocation
//
PointerPte = MiAddressToPte(BaseVa);
ASSERT(PointerPte->u.Hard.Valid == 1);
//
// Grab the PFN NextEntry and index
//
Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
//
// Now mark it as the beginning of an allocation
//
ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as special pool if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK)
{
Pfn1->u4.VerifierAllocation = 1;
}
//
// Check if the allocation is larger than one page
//
if (SizeInPages != 1)
{
//
// Navigate to the last PFN entry and PTE
//
PointerPte += SizeInPages - 1;
ASSERT(PointerPte->u.Hard.Valid == 1);
Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
}
//
// Mark this PFN as the last (might be the same as the first)
//
ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Release the nonpaged pool lock, and return the allocation
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
return BaseVa;
}
//
// Try the next free page entry
//
NextEntry = FreeEntry->List.Flink;
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
} while (++NextHead < LastHead);
//
// If we got here, we're out of space.
// Start by releasing the lock
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Allocate some system PTEs
//
StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
PointerPte = StartPte;
if (StartPte == NULL)
{
//
// Ran out of memory
//
DPRINT1("Out of NP Expansion Pool\n");
return NULL;
}
//
// Acquire the pool lock now
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
//
// Lock the PFN database too
//
LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
//
// Loop the pages
//
TempPte = ValidKernelPte;
do
{
/* Allocate a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->PteAddress = PointerPte;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u4.VerifierAllocation = 0;
/* Write the PTE for it */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
MI_WRITE_VALID_PTE(PointerPte++, TempPte);
} while (--SizeInPages > 0);
//
// This is the last page
//
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Get the first page and mark it as such
//
Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as a verifier allocation if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1;
//
// Release the PFN and nonpaged pool lock
//
KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Return the address
//
return MiPteToAddress(StartPte);
}
VOID
MiReleaseAllMemory (
VOID
)
/*++
Routine Description:
This function performs the final release of memory management allocations.
Arguments:
None.
Return Value:
None.
Environment:
No references to paged pool or pageable code/data are allowed.
--*/
{
ULONG i;
ULONG j;
PEVENT_COUNTER EventSupport;
PUNLOADED_DRIVERS Entry;
PLIST_ENTRY NextEntry;
PKLDR_DATA_TABLE_ENTRY DataTableEntry;
PLOAD_IMPORTS ImportList;
PMI_VERIFIER_DRIVER_ENTRY Verifier;
PMMINPAGE_SUPPORT Support;
PSLIST_ENTRY SingleListEntry;
PDRIVER_SPECIFIED_VERIFIER_THUNKS ThunkTableBase;
PMMMOD_WRITER_MDL_ENTRY ModWriterEntry;
ASSERT (MmUnusedSegmentList.Flink == &MmUnusedSegmentList);
//
// Don't clear free pages so problems can be debugged.
//
MiZeroingDisabled = TRUE;
//
// Free the unloaded driver list.
//
if (MmUnloadedDrivers != NULL) {
Entry = &MmUnloadedDrivers[0];
for (i = 0; i < MI_UNLOADED_DRIVERS; i += 1) {
if (Entry->Name.Buffer != NULL) {
RtlFreeUnicodeString (&Entry->Name);
}
Entry += 1;
}
ExFreePool (MmUnloadedDrivers);
}
NextEntry = MmLoadedUserImageList.Flink;
while (NextEntry != &MmLoadedUserImageList) {
DataTableEntry = CONTAINING_RECORD (NextEntry,
KLDR_DATA_TABLE_ENTRY,
InLoadOrderLinks);
NextEntry = NextEntry->Flink;
ExFreePool ((PVOID)DataTableEntry);
}
//
// Release the loaded module list entries.
//
NextEntry = PsLoadedModuleList.Flink;
while (NextEntry != &PsLoadedModuleList) {
DataTableEntry = CONTAINING_RECORD (NextEntry,
KLDR_DATA_TABLE_ENTRY,
InLoadOrderLinks);
ImportList = (PLOAD_IMPORTS)DataTableEntry->LoadedImports;
if ((ImportList != (PVOID)LOADED_AT_BOOT) &&
(ImportList != (PVOID)NO_IMPORTS_USED) &&
(!SINGLE_ENTRY(ImportList))) {
ExFreePool (ImportList);
}
if (DataTableEntry->FullDllName.Buffer != NULL) {
ASSERT (DataTableEntry->FullDllName.Buffer == DataTableEntry->BaseDllName.Buffer);
}
NextEntry = NextEntry->Flink;
ExFreePool ((PVOID)DataTableEntry);
}
//
// Free the physical memory descriptor block.
//
ExFreePool (MmPhysicalMemoryBlock);
ExFreePool (MiPfnBitMap.Buffer);
//
// Free the system views structure.
//
if (MmSession.SystemSpaceViewTable != NULL) {
ExFreePool (MmSession.SystemSpaceViewTable);
}
if (MmSession.SystemSpaceBitMap != NULL) {
ExFreePool (MmSession.SystemSpaceBitMap);
}
//
// Free the pagefile structures - note the PageFileName buffer was freed
// earlier as it resided in paged pool and may have needed an inpage
// to be freed.
//
for (i = 0; i < MmNumberOfPagingFiles; i += 1) {
ASSERT (MmPagingFile[i]->PageFileName.Buffer == NULL);
for (j = 0; j < MM_PAGING_FILE_MDLS; j += 1) {
ExFreePool (MmPagingFile[i]->Entry[j]);
}
ExFreePool (MmPagingFile[i]->Bitmap);
ExFreePool (MmPagingFile[i]);
}
ASSERT (MmNumberOfMappedMdlsInUse == 0);
i = 0;
while (IsListEmpty (&MmMappedFileHeader.ListHead) != 0) {
ModWriterEntry = (PMMMOD_WRITER_MDL_ENTRY)RemoveHeadList (
&MmMappedFileHeader.ListHead);
ExFreePool (ModWriterEntry);
i += 1;
}
ASSERT (i == MmNumberOfMappedMdls);
//
// Free the paged pool bitmaps.
//
ExFreePool (MmPagedPoolInfo.PagedPoolAllocationMap);
ExFreePool (MmPagedPoolInfo.EndOfPagedPoolBitmap);
if (VerifierLargePagedPoolMap != NULL) {
ExFreePool (VerifierLargePagedPoolMap);
}
//
// Free the inpage structures.
//
while (ExQueryDepthSList (&MmInPageSupportSListHead) != 0) {
SingleListEntry = InterlockedPopEntrySList (&MmInPageSupportSListHead);
if (SingleListEntry != NULL) {
Support = CONTAINING_RECORD (SingleListEntry,
MMINPAGE_SUPPORT,
ListEntry);
ASSERT (Support->u1.e1.PrefetchMdlHighBits == 0);
ExFreePool (Support);
}
}
while (ExQueryDepthSList (&MmEventCountSListHead) != 0) {
EventSupport = (PEVENT_COUNTER) InterlockedPopEntrySList (&MmEventCountSListHead);
if (EventSupport != NULL) {
ExFreePool (EventSupport);
}
}
//
// Free the verifier list last because it must be consulted to debug
// any bugchecks.
//
NextEntry = MiVerifierDriverAddedThunkListHead.Flink;
if (NextEntry != NULL) {
while (NextEntry != &MiVerifierDriverAddedThunkListHead) {
ThunkTableBase = CONTAINING_RECORD (NextEntry,
DRIVER_SPECIFIED_VERIFIER_THUNKS,
ListEntry );
NextEntry = NextEntry->Flink;
ExFreePool (ThunkTableBase);
}
}
NextEntry = MiSuspectDriverList.Flink;
while (NextEntry != &MiSuspectDriverList) {
Verifier = CONTAINING_RECORD(NextEntry,
MI_VERIFIER_DRIVER_ENTRY,
Links);
NextEntry = NextEntry->Flink;
ExFreePool (Verifier);
}
}
