VOID
NTAPI
ExSwapinWorkerThreads(IN BOOLEAN AllowSwap)
{
KEVENT Event;
PETHREAD CurrentThread = PsGetCurrentThread(), Thread;
PEPROCESS Process = PsInitialSystemProcess;
KAPC Apc;
PAGED_CODE();
/* Initialize an event so we know when we're done */
KeInitializeEvent(&Event, NotificationEvent, FALSE);
/* Lock this routine */
ExAcquireFastMutex(&ExpWorkerSwapinMutex);
/* New threads cannot swap anymore */
ExpWorkersCanSwap = AllowSwap;
/* Loop all threads in the system process */
Thread = PsGetNextProcessThread(Process, NULL);
while (Thread)
{
/* Skip threads with explicit permission to do this */
if (Thread->ExWorkerCanWaitUser) goto Next;
/* Check if we reached ourselves */
if (Thread == CurrentThread)
{
/* Do it inline */
KeSetKernelStackSwapEnable(AllowSwap);
}
else
{
/* Queue an APC */
KeInitializeApc(&Apc,
&Thread->Tcb,
InsertApcEnvironment,
ExpSetSwappingKernelApc,
NULL,
NULL,
KernelMode,
&AllowSwap);
if (KeInsertQueueApc(&Apc, &Event, NULL, 3))
{
/* Wait for the APC to run */
KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL);
KeClearEvent(&Event);
}
}
/* Next thread */
Next:
Thread = PsGetNextProcessThread(Process, Thread);
}
/* Release the lock */
ExReleaseFastMutex(&ExpWorkerSwapinMutex);
}
VOID
ExpShutdownWorker (
IN PVOID Parameter
)
{
PETHREAD CurrentThread;
PSHUTDOWN_WORK_ITEM ShutdownItem;
ShutdownItem = (PSHUTDOWN_WORK_ITEM) Parameter;
ASSERT (ShutdownItem != NULL);
if (ShutdownItem->PrevThread != NULL) {
//
// Wait for the previous thread to exit -- if it's in the same
// queue, it probably has already, but we need to make sure
// (and if it's not, we *definitely* need to make sure).
//
KeWaitForSingleObject (ShutdownItem->PrevThread,
Executive,
KernelMode,
FALSE,
NULL);
ObDereferenceObject (ShutdownItem->PrevThread);
ShutdownItem->PrevThread = NULL;
}
//
// Decrement the worker count.
//
InterlockedDecrement (&ExWorkerQueue[ShutdownItem->QueueType].Info.QueueWorkerInfo);
CurrentThread = PsGetCurrentThread();
if ((!ExpCheckQueueShutdown(DelayedWorkQueue, ShutdownItem)) &&
(!ExpCheckQueueShutdown(CriticalWorkQueue, ShutdownItem))) {
//
// We're the last worker to exit
//
ASSERT (!ExpLastWorkerThread);
ExpLastWorkerThread = CurrentThread;
ObReferenceObject (ExpLastWorkerThread);
KeSetEvent (&ExpThreadSetManagerShutdownEvent, 0, FALSE);
}
KeSetKernelStackSwapEnable (TRUE);
CurrentThread->ActiveExWorker = 0;
PsTerminateSystemThread (STATUS_SYSTEM_SHUTDOWN);
}
VOID
NTAPI
ExpSetSwappingKernelApc(IN PKAPC Apc,
OUT PKNORMAL_ROUTINE *NormalRoutine,
IN OUT PVOID *NormalContext,
IN OUT PVOID *SystemArgument1,
IN OUT PVOID *SystemArgument2)
{
PBOOLEAN AllowSwap;
PKEVENT Event = (PKEVENT)*SystemArgument1;
/* Make sure it's an active worker */
if (PsGetCurrentThread()->ActiveExWorker)
{
/* Read the setting from the context flag */
AllowSwap = (PBOOLEAN)NormalContext;
KeSetKernelStackSwapEnable(*AllowSwap);
}
/* Let caller know that we're done */
KeSetEvent(Event, 0, FALSE);
}
VOID
ExpSetSwappingKernelApc (
IN PKAPC Apc,
OUT PKNORMAL_ROUTINE *NormalRoutine,
IN OUT PVOID NormalContext,
IN OUT PVOID *SystemArgument1,
IN OUT PVOID *SystemArgument2
)
{
PBOOLEAN AllowSwap;
PKEVENT SwapSetEvent;
UNREFERENCED_PARAMETER (Apc);
UNREFERENCED_PARAMETER (NormalRoutine);
UNREFERENCED_PARAMETER (SystemArgument2);
//
// SystemArgument1 is a pointer to the event to signal once this
// thread has finished servicing the request.
//
SwapSetEvent = (PKEVENT) *SystemArgument1;
//
// Don't disable stack swapping if the thread is exiting because
// it cannot exit this way without bugchecking. Skip it on enables
// too since the thread is bailing anyway.
//
if (PsGetCurrentThread()->ActiveExWorker != 0) {
AllowSwap = NormalContext;
KeSetKernelStackSwapEnable (*AllowSwap);
}
KeSetEvent (SwapSetEvent, 0, FALSE);
}
/*++
* @name ExpWorkerThreadEntryPoint
*
* The ExpWorkerThreadEntryPoint routine is the entrypoint for any new
* worker thread created by teh system.
*
* @param Context
* Contains the work queue type masked with a flag specifing whether the
* thread is dynamic or not.
*
* @return None.
*
* @remarks A dynamic thread can timeout after 10 minutes of waiting on a queue
* while a static thread will never timeout.
*
* Worker threads must return at IRQL == PASSIVE_LEVEL, must not have
* active impersonation info, and must not have disabled APCs.
*
* NB: We will re-enable APCs for broken threads but all other cases
* will generate a bugcheck.
*
*--*/
VOID
NTAPI
ExpWorkerThreadEntryPoint(IN PVOID Context)
{
PWORK_QUEUE_ITEM WorkItem;
PLIST_ENTRY QueueEntry;
WORK_QUEUE_TYPE WorkQueueType;
PEX_WORK_QUEUE WorkQueue;
LARGE_INTEGER Timeout;
PLARGE_INTEGER TimeoutPointer = NULL;
PETHREAD Thread = PsGetCurrentThread();
KPROCESSOR_MODE WaitMode;
EX_QUEUE_WORKER_INFO OldValue, NewValue;
/* Check if this is a dyamic thread */
if ((ULONG_PTR)Context & EX_DYNAMIC_WORK_THREAD)
{
/* It is, which means we will eventually time out after 10 minutes */
Timeout.QuadPart = Int32x32To64(10, -10000000 * 60);
TimeoutPointer = &Timeout;
}
/* Get Queue Type and Worker Queue */
WorkQueueType = (WORK_QUEUE_TYPE)((ULONG_PTR)Context &
~EX_DYNAMIC_WORK_THREAD);
WorkQueue = &ExWorkerQueue[WorkQueueType];
/* Select the wait mode */
WaitMode = (UCHAR)WorkQueue->Info.WaitMode;
/* Nobody should have initialized this yet, do it now */
ASSERT(Thread->ExWorkerCanWaitUser == 0);
if (WaitMode == UserMode) Thread->ExWorkerCanWaitUser = TRUE;
/* If we shouldn't swap, disable that feature */
if (!ExpWorkersCanSwap) KeSetKernelStackSwapEnable(FALSE);
/* Set the worker flags */
do
{
/* Check if the queue is being disabled */
if (WorkQueue->Info.QueueDisabled)
{
/* Re-enable stack swapping and kill us */
KeSetKernelStackSwapEnable(TRUE);
PsTerminateSystemThread(STATUS_SYSTEM_SHUTDOWN);
}
/* Increase the worker count */
OldValue = WorkQueue->Info;
NewValue = OldValue;
NewValue.WorkerCount++;
}
while (InterlockedCompareExchange((PLONG)&WorkQueue->Info,
*(PLONG)&NewValue,
*(PLONG)&OldValue) != *(PLONG)&OldValue);
/* Success, you are now officially a worker thread! */
Thread->ActiveExWorker = TRUE;
/* Loop forever */
ProcessLoop:
for (;;)
{
/* Wait for Something to Happen on the Queue */
QueueEntry = KeRemoveQueue(&WorkQueue->WorkerQueue,
WaitMode,
TimeoutPointer);
/* Check if we timed out and quit this loop in that case */
if ((NTSTATUS)(ULONG_PTR)QueueEntry == STATUS_TIMEOUT) break;
/* Increment Processed Work Items */
InterlockedIncrement((PLONG)&WorkQueue->WorkItemsProcessed);
/* Get the Work Item */
WorkItem = CONTAINING_RECORD(QueueEntry, WORK_QUEUE_ITEM, List);
/* Make sure nobody is trying to play smart with us */
ASSERT((ULONG_PTR)WorkItem->WorkerRoutine > MmUserProbeAddress);
/* Call the Worker Routine */
WorkItem->WorkerRoutine(WorkItem->Parameter);
/* Make sure APCs are not disabled */
if (Thread->Tcb.SpecialApcDisable)
{
/* We're nice and do it behind your back */
DPRINT1("Warning: Broken Worker Thread: %p %lx %p came back "
"with APCs disabled!\n",
WorkItem->WorkerRoutine,
WorkItem->Parameter,
WorkItem);
Thread->Tcb.SpecialApcDisable = 0;
}
/* Make sure it returned at right IRQL */
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
{
/* It didn't, bugcheck! */
KeBugCheckEx(WORKER_THREAD_RETURNED_AT_BAD_IRQL,
(ULONG_PTR)WorkItem->WorkerRoutine,
KeGetCurrentIrql(),
(ULONG_PTR)WorkItem->Parameter,
(ULONG_PTR)WorkItem);
}
/* Make sure it returned with Impersionation Disabled */
if (Thread->ActiveImpersonationInfo)
{
/* It didn't, bugcheck! */
KeBugCheckEx(IMPERSONATING_WORKER_THREAD,
(ULONG_PTR)WorkItem->WorkerRoutine,
(ULONG_PTR)WorkItem->Parameter,
(ULONG_PTR)WorkItem,
0);
}
}
/* This is a dynamic thread. Terminate it unless IRPs are pending */
if (!IsListEmpty(&Thread->IrpList)) goto ProcessLoop;
/* Don't terminate it if the queue is disabled either */
if (WorkQueue->Info.QueueDisabled) goto ProcessLoop;
/* Set the worker flags */
do
{
/* Decrease the worker count */
OldValue = WorkQueue->Info;
NewValue = OldValue;
NewValue.WorkerCount--;
}
while (InterlockedCompareExchange((PLONG)&WorkQueue->Info,
*(PLONG)&NewValue,
*(PLONG)&OldValue) != *(PLONG)&OldValue);
/* Decrement dynamic thread count */
InterlockedDecrement(&WorkQueue->DynamicThreadCount);
/* We're not a worker thread anymore */
Thread->ActiveExWorker = FALSE;
/* Re-enable the stack swap */
KeSetKernelStackSwapEnable(TRUE);
return;
}
VOID
ExpWorkerThread (
IN PVOID StartContext
)
{
PLIST_ENTRY Entry;
WORK_QUEUE_TYPE QueueType;
PWORK_QUEUE_ITEM WorkItem;
KPROCESSOR_MODE WaitMode;
LARGE_INTEGER TimeoutValue;
PLARGE_INTEGER Timeout;
PETHREAD Thread;
PEX_WORK_QUEUE WorkerQueue;
PWORKER_THREAD_ROUTINE WorkerRoutine;
PVOID Parameter;
EX_QUEUE_WORKER_INFO OldWorkerInfo;
EX_QUEUE_WORKER_INFO NewWorkerInfo;
ULONG CountForQueueEmpty;
//
// Set timeout value etc according to whether we are static or dynamic.
//
if (((ULONG_PTR)StartContext & DYNAMIC_WORKER_THREAD) == 0) {
//
// We are being created as a static thread. As such it will not
// terminate, so there is no point in timing out waiting for a work
// item.
//
Timeout = NULL;
}
else {
//
// This is a dynamic worker thread. It has a non-infinite timeout
// so that it can eventually terminate.
//
TimeoutValue.QuadPart = -DYNAMIC_THREAD_TIMEOUT;
Timeout = &TimeoutValue;
}
Thread = PsGetCurrentThread ();
//
// If the thread is a critical worker thread, then set the thread
// priority to the lowest realtime level. Otherwise, set the base
// thread priority to time critical.
//
QueueType = (WORK_QUEUE_TYPE)
((ULONG_PTR)StartContext & ~DYNAMIC_WORKER_THREAD);
WorkerQueue = &ExWorkerQueue[QueueType];
WaitMode = (KPROCESSOR_MODE) WorkerQueue->Info.WaitMode;
ASSERT (Thread->ExWorkerCanWaitUser == 0);
if (WaitMode == UserMode) {
Thread->ExWorkerCanWaitUser = 1;
}
#if defined(REMOTE_BOOT)
//
// In diskless NT scenarios ensure that the kernel stack of the worker
// threads will not be swapped out.
//
if (IoRemoteBootClient) {
KeSetKernelStackSwapEnable (FALSE);
}
#endif // defined(REMOTE_BOOT)
//
// Register as a worker, exiting if the queue's going down and
// there aren't any workers in the queue to hand us the shutdown
// work item if we enter the queue (we want to be able to enter a
// queue even if the queue's shutting down, in case there's a
// backlog of work items that the balance manager thread's decided
// we should be helping to process).
//
if (PO_SHUTDOWN_QUEUE == QueueType) {
CountForQueueEmpty = 1;
}
else {
CountForQueueEmpty = 0;
}
if (ExpWorkersCanSwap == FALSE) {
KeSetKernelStackSwapEnable (FALSE);
}
do {
OldWorkerInfo.QueueWorkerInfo = ReadForWriteAccess (&WorkerQueue->Info.QueueWorkerInfo);
if (OldWorkerInfo.QueueDisabled &&
OldWorkerInfo.WorkerCount <= CountForQueueEmpty) {
//
// The queue is disabled and empty so just exit.
//
KeSetKernelStackSwapEnable (TRUE);
PsTerminateSystemThread (STATUS_SYSTEM_SHUTDOWN);
}
NewWorkerInfo.QueueWorkerInfo = OldWorkerInfo.QueueWorkerInfo;
NewWorkerInfo.WorkerCount += 1;
} while (OldWorkerInfo.QueueWorkerInfo !=
InterlockedCompareExchange (&WorkerQueue->Info.QueueWorkerInfo,
NewWorkerInfo.QueueWorkerInfo,
OldWorkerInfo.QueueWorkerInfo));
//
// As of this point, we must only exit if we decrement the worker
// count without the queue disabled flag being set. (Unless we
// exit due to the shutdown work item, which also decrements the
// worker count).
//
Thread->ActiveExWorker = 1;
//
// Loop forever waiting for a work queue item, calling the processing
// routine, and then waiting for another work queue item.
//
do {
//
// Wait until something is put in the queue or until we time out.
//
// By specifying a wait mode of UserMode, the thread's kernel
// stack is swappable.
//
Entry = KeRemoveQueue (&WorkerQueue->WorkerQueue,
WaitMode,
Timeout);
if ((ULONG_PTR)Entry != STATUS_TIMEOUT) {
//
// This is a real work item, process it.
//
// Update the total number of work items processed.
//
InterlockedIncrement ((PLONG)&WorkerQueue->WorkItemsProcessed);
WorkItem = CONTAINING_RECORD(Entry, WORK_QUEUE_ITEM, List);
WorkerRoutine = WorkItem->WorkerRoutine;
Parameter = WorkItem->Parameter;
//
// Catch worker routines referencing a user mode address.
//
ASSERT ((ULONG_PTR)WorkerRoutine > MmUserProbeAddress);
//
// Execute the specified routine.
//
((PWORKER_THREAD_ROUTINE)WorkerRoutine) (Parameter);
#if DBG
if (IsListEmpty (&Thread->IrpList)) {
//
// See if a worker just returned while holding a resource
//
ExCheckIfResourceOwned ();
}
#endif
//
// Catch worker routines that forget to leave a critial/guarded
// region. In the debug case execute a breakpoint. In the free
// case zero the flag so that APCs can continue to fire to this
// thread.
//
if (Thread->Tcb.CombinedApcDisable != 0) {
DbgPrint ((char*)ExpWorkerApcDisabledMessage,
WorkerRoutine,
Parameter,
WorkItem);
ASSERT (FALSE);
Thread->Tcb.CombinedApcDisable = 0;
}
if (KeGetCurrentIrql () != PASSIVE_LEVEL) {
KeBugCheckEx (WORKER_THREAD_RETURNED_AT_BAD_IRQL,
(ULONG_PTR)WorkerRoutine,
(ULONG_PTR)KeGetCurrentIrql(),
(ULONG_PTR)Parameter,
(ULONG_PTR)WorkItem);
}
if (PS_IS_THREAD_IMPERSONATING (Thread)) {
KeBugCheckEx (IMPERSONATING_WORKER_THREAD,
(ULONG_PTR)WorkerRoutine,
(ULONG_PTR)Parameter,
(ULONG_PTR)WorkItem,
0);
}
continue;
}
//
// These things are known:
//
// - Static worker threads do not time out, so this is a dynamic
// worker thread.
//
// - This thread has been waiting for a long time with nothing
// to do.
//
if (IsListEmpty (&Thread->IrpList) == FALSE) {
//
// There is still I/O pending, can't terminate yet.
//
continue;
}
//
// Get out of the queue, if we can
//
do {
OldWorkerInfo.QueueWorkerInfo = ReadForWriteAccess (&WorkerQueue->Info.QueueWorkerInfo);
if (OldWorkerInfo.QueueDisabled) {
//
// We're exiting via the queue disable work item;
// there's no point in expiring here.
//
break;
}
NewWorkerInfo.QueueWorkerInfo = OldWorkerInfo.QueueWorkerInfo;
NewWorkerInfo.WorkerCount -= 1;
} while (OldWorkerInfo.QueueWorkerInfo
!= InterlockedCompareExchange(&WorkerQueue->Info.QueueWorkerInfo,
NewWorkerInfo.QueueWorkerInfo,
OldWorkerInfo.QueueWorkerInfo));
if (OldWorkerInfo.QueueDisabled) {
//
// We're exiting via the queue disable work item
//
continue;
}
//
// This dynamic thread can be terminated.
//
break;
} while (TRUE);
//
// Terminate this dynamic thread.
//
InterlockedDecrement ((PLONG)&WorkerQueue->DynamicThreadCount);
//
// Carefully clear this before marking the thread stack as swap enabled
// so that an incoming APC won't inadvertently disable the stack swap
// afterwards.
//
Thread->ActiveExWorker = 0;
//
// We will bugcheck if we terminate a thread with stack swapping
// disabled.
//
KeSetKernelStackSwapEnable (TRUE);
return;
}
VOID
ExSwapinWorkerThreads (
IN BOOLEAN AllowSwap
)
/*++
Routine Description:
Sets the kernel stacks of the delayed worker threads to be swappable
or pins them into memory.
Arguments:
AllowSwap - Supplies TRUE if worker kernel stacks should be swappable,
FALSE if not.
Return Value:
None.
--*/
{
PETHREAD Thread;
PETHREAD CurrentThread;
PEPROCESS Process;
KAPC Apc;
KEVENT SwapSetEvent;
PAGED_CODE();
CurrentThread = PsGetCurrentThread();
KeInitializeEvent (&SwapSetEvent,
NotificationEvent,
FALSE);
Process = PsInitialSystemProcess;
//
// Serialize callers.
//
ExAcquireFastMutex (&ExpWorkerSwapinMutex);
//
// Stop new threads from swapping.
//
ExpWorkersCanSwap = AllowSwap;
//
// Stop existing worker threads from swapping.
//
for (Thread = PsGetNextProcessThread (Process, NULL);
Thread != NULL;
Thread = PsGetNextProcessThread (Process, Thread)) {
//
// Skip threads that are not worker threads or worker threads that
// were permanently marked noswap at creation time.
//
if (Thread->ExWorkerCanWaitUser == 0) {
continue;
}
if (Thread == CurrentThread) {
//
// No need to use an APC on the current thread.
//
KeSetKernelStackSwapEnable (AllowSwap);
}
else {
//
// Queue an APC to the thread, and wait for it to fire:
//
KeInitializeApc (&Apc,
&Thread->Tcb,
InsertApcEnvironment,
ExpSetSwappingKernelApc,
NULL,
NULL,
KernelMode,
&AllowSwap);
if (KeInsertQueueApc (&Apc, &SwapSetEvent, NULL, 3)) {
KeWaitForSingleObject (&SwapSetEvent,
Executive,
KernelMode,
FALSE,
NULL);
KeClearEvent(&SwapSetEvent);
}
}
}
ExReleaseFastMutex (&ExpWorkerSwapinMutex);
}
