PKTHREAD
FASTCALL
KiFindReadyThread (
IN ULONG Processor,
IN KPRIORITY LowPriority
)
/*++
Routine Description:
This function searches the dispatcher ready queues from the specified
high priority to the specified low priority in an attempt to find a thread
that can execute on the specified processor.
Arguments:
Processor - Supplies the number of the processor to find a thread for.
LowPriority - Supplies the lowest priority dispatcher ready queue to
examine.
Return Value:
If a thread is located that can execute on the specified processor, then
the address of the thread object is returned. Otherwise a null pointer is
returned.
--*/
{
ULONG HighPriority;
PRLIST_ENTRY ListHead;
PRLIST_ENTRY NextEntry;
ULONG PrioritySet;
KAFFINITY ProcessorSet;
PRKTHREAD Thread;
PRKTHREAD Thread1;
ULONG TickLow;
ULONG WaitTime;
//
// Compute the set of priority levels that should be scanned in an attempt
// to find a thread that can run on the specified processor.
//
PrioritySet = (~((1 << LowPriority) - 1)) & KiReadySummary;
#if !defined(NT_UP)
ProcessorSet = (KAFFINITY)(1 << Processor);
#endif
FindFirstSetLeftMember(PrioritySet, &HighPriority);
ListHead = &KiDispatcherReadyListHead[HighPriority];
PrioritySet <<= (31 - HighPriority);
while (PrioritySet != 0) {
//
// If the next bit in the priority set is a one, then examine the
// corresponding dispatcher ready queue.
//
if ((LONG)PrioritySet < 0) {
NextEntry = ListHead->Flink;
ASSERT(NextEntry != ListHead);
#if defined(NT_UP)
Thread = CONTAINING_RECORD(NextEntry, KTHREAD, WaitListEntry);
RemoveEntryList(&Thread->WaitListEntry);
if (IsListEmpty(ListHead)) {
ClearMember(HighPriority, KiReadySummary);
}
return (PKTHREAD)Thread;
#else
//
// Scan the specified dispatcher ready queue for a suitable
// thread to execute.
//
while (NextEntry != ListHead) {
Thread = CONTAINING_RECORD(NextEntry, KTHREAD, WaitListEntry);
NextEntry = NextEntry->Flink;
if (Thread->Affinity & ProcessorSet) {
//
// If the found thread ran on the specified processor
// last, the processor is the ideal processor for the
// thread, the thread has been waiting for longer than
// a quantum, or its priority is greater than low realtime
// plus 8, then the selected thread is returned. Otherwise,
// an attempt is made to find a more appropriate thread.
//
TickLow = KiQueryLowTickCount();
WaitTime = TickLow - Thread->WaitTime;
if ((KiThreadSelectNotifyRoutine ?
(KiThreadSelectNotifyRoutine(((PETHREAD)Thread)->Cid.UniqueThread) == FALSE) :
(((ULONG)Thread->NextProcessor != Processor) &&
((ULONG)Thread->IdealProcessor != Processor))) &&
(WaitTime < (READY_SKIP_QUANTUM + 1)) &&
(HighPriority < (LOW_REALTIME_PRIORITY + 9))) {
//
// Search forward in the ready queue until the end
// of the list is reached or a more appropriate
// thread is found.
//
while (NextEntry != ListHead) {
Thread1 = CONTAINING_RECORD(NextEntry,
KTHREAD,
WaitListEntry);
NextEntry = NextEntry->Flink;
if ((Thread1->Affinity & ProcessorSet) &&
(KiThreadSelectNotifyRoutine ?
(KiThreadSelectNotifyRoutine(((PETHREAD)Thread)->Cid.UniqueThread) != FALSE) :
(((ULONG)Thread1->NextProcessor == Processor) ||
((ULONG)Thread1->IdealProcessor == Processor)))) {
Thread = Thread1;
break;
}
WaitTime = TickLow - Thread1->WaitTime;
if (WaitTime >= (READY_SKIP_QUANTUM + 1)) {
break;
}
}
}
if (Processor == (ULONG)Thread->IdealProcessor) {
KiIncrementSwitchCounter(FindIdeal);
} else if (Processor == (ULONG)Thread->NextProcessor) {
KiIncrementSwitchCounter(FindLast);
} else {
KiIncrementSwitchCounter(FindAny);
}
Thread->NextProcessor = (CCHAR)Processor;
RemoveEntryList(&Thread->WaitListEntry);
if (IsListEmpty(ListHead)) {
ClearMember(HighPriority, KiReadySummary);
}
return (PKTHREAD)Thread;
}
}
#endif
}
HighPriority -= 1;
ListHead -= 1;
PrioritySet <<= 1;
};
//
// No thread could be found, return a null pointer.
//
return (PKTHREAD)NULL;
}
VOID
KiTimerExpiration (
IN PKDPC TimerDpc,
IN PVOID DeferredContext,
IN PVOID SystemArgument1,
IN PVOID SystemArgument2
)
/*++
Routine Description:
This function is called when the clock interupt routine discovers that
a timer has expired.
Arguments:
TimerDpc - Supplies a pointer to a control object of type DPC.
DeferredContext - Not used.
SystemArgument1 - Supplies the starting timer table index value to
use for the timer table scan.
SystemArgument2 - Not used.
Return Value:
None.
--*/
{
ULARGE_INTEGER CurrentTime;
LIST_ENTRY ExpiredListHead;
LONG HandLimit;
LONG Index;
PLIST_ENTRY ListHead;
PLIST_ENTRY NextEntry;
KIRQL OldIrql;
PKTIMER Timer;
//
// Acquire the dispatcher database lock and read the current interrupt
// time to determine which timers have expired.
//
KiLockDispatcherDatabase(&OldIrql);
KiQueryInterruptTime((PLARGE_INTEGER)&CurrentTime);
//
// If the timer table has not wrapped, then start with the specified
// timer table index value, and scan for timer entries that have expired.
// Otherwise, start with the first entry in the timer table and scan the
// entire table for timer entries that have expired.
//
// N.B. This later condition exists when DPC processing is blocked for a
// period longer than one round trip throught the timer table.
//
HandLimit = (LONG)KiQueryLowTickCount();
if (((ULONG)(HandLimit - PtrToLong(SystemArgument1))) >= TIMER_TABLE_SIZE) {
Index = - 1;
HandLimit = TIMER_TABLE_SIZE - 1;
} else {
Index = (PtrToLong(SystemArgument1) - 1) & (TIMER_TABLE_SIZE - 1);
HandLimit &= (TIMER_TABLE_SIZE - 1);
}
InitializeListHead(&ExpiredListHead);
do {
Index = (Index + 1) & (TIMER_TABLE_SIZE - 1);
ListHead = &KiTimerTableListHead[Index];
NextEntry = ListHead->Flink;
while (NextEntry != ListHead) {
Timer = CONTAINING_RECORD(NextEntry, KTIMER, TimerListEntry);
if (Timer->DueTime.QuadPart <= CurrentTime.QuadPart) {
//
// The next timer in the current timer list has expired.
// Remove the entry from the timer list and insert the
// timer in the expired list.
//
RemoveEntryList(&Timer->TimerListEntry);
InsertTailList(&ExpiredListHead, &Timer->TimerListEntry);
NextEntry = ListHead->Flink;
} else {
break;
}
}
} while(Index != HandLimit);
#if DBG
if ((PtrToUlong(SystemArgument2) == 0) && (KeNumberProcessors == 1)) {
KiCheckTimerTable(CurrentTime);
}
#endif
//
// Process the expired timer list.
//
// N.B. The following function returns with the dispatcher database
// unlocked.
//
KiTimerListExpire(&ExpiredListHead, OldIrql);
return;
}
VOID
FASTCALL
KiReadyThread (
IN PRKTHREAD Thread
)
/*++
Routine Description:
This function readies a thread for execution and attempts to immediately
dispatch the thread for execution by preempting another lower priority
thread. If a thread can be preempted, then the specified thread enters
the standby state and the target processor is requested to dispatch. If
another thread cannot be preempted, then the specified thread is inserted
either at the head or tail of the dispatcher ready selected by its priority
acccording to whether it was preempted or not.
Arguments:
Thread - Supplies a pointer to a dispatcher object of type thread.
Return Value:
None.
--*/
{
PRKPRCB Prcb;
BOOLEAN Preempted;
KPRIORITY Priority;
PRKPROCESS Process;
ULONG Processor;
KPRIORITY ThreadPriority;
PRKTHREAD Thread1;
KAFFINITY IdleSet;
//
// Save value of thread's preempted flag, set thread preempted FALSE,
// capture the thread priority, and set clear the read wait time.
//
Preempted = Thread->Preempted;
Thread->Preempted = FALSE;
ThreadPriority = Thread->Priority;
Thread->WaitTime = KiQueryLowTickCount();
//
// If the thread's process is not in memory, then insert the thread in
// the process ready queue and inswap the process.
//
Process = Thread->ApcState.Process;
if (Process->State != ProcessInMemory) {
Thread->State = Ready;
Thread->ProcessReadyQueue = TRUE;
InsertTailList(&Process->ReadyListHead, &Thread->WaitListEntry);
if (Process->State == ProcessOutOfMemory) {
Process->State = ProcessInTransition;
InsertTailList(&KiProcessInSwapListHead, &Process->SwapListEntry);
KiSwapEvent.Header.SignalState = 1;
if (IsListEmpty(&KiSwapEvent.Header.WaitListHead) == FALSE) {
KiWaitTest(&KiSwapEvent, BALANCE_INCREMENT);
}
}
return;
} else if (Thread->KernelStackResident == FALSE) {
//
// The thread's kernel stack is not resident. Increment the process
// stack count, set the state of the thread to transition, insert
// the thread in the kernel stack inswap list, and set the kernel
// stack inswap event.
//
Process->StackCount += 1;
Thread->State = Transition;
InsertTailList(&KiStackInSwapListHead, &Thread->WaitListEntry);
KiSwapEvent.Header.SignalState = 1;
if (IsListEmpty(&KiSwapEvent.Header.WaitListHead) == FALSE) {
KiWaitTest(&KiSwapEvent, BALANCE_INCREMENT);
}
return;
} else {
//
// If there is an idle processor, then schedule the thread on an
// idle processor giving preference to the processor the thread
// last ran on. Otherwise, try to preempt either a thread in the
// standby or running state.
//
#if defined(NT_UP)
Prcb = KiProcessorBlock[0];
if (KiIdleSummary != 0) {
KiIdleSummary = 0;
KiIncrementSwitchCounter(IdleLast);
#else
IdleSet = KiIdleSummary & Thread->Affinity;
if (IdleSet != 0) {
Processor = Thread->IdealProcessor;
if ((IdleSet & (1 << Processor)) == 0) {
Processor = Thread->NextProcessor;
if ((IdleSet & (1 << Processor)) == 0) {
Prcb = KeGetCurrentPrcb();
if ((IdleSet & Prcb->SetMember) == 0) {
FindFirstSetLeftMember(IdleSet, &Processor);
KiIncrementSwitchCounter(IdleAny);
} else {
Processor = Prcb->Number;
KiIncrementSwitchCounter(IdleCurrent);
}
} else {
KiIncrementSwitchCounter(IdleLast);
}
} else {
KiIncrementSwitchCounter(IdleIdeal);
}
Thread->NextProcessor = (CCHAR)Processor;
ClearMember(Processor, KiIdleSummary);
Prcb = KiProcessorBlock[Processor];
#endif
Prcb->NextThread = Thread;
Thread->State = Standby;
return;
} else {
#if !defined(NT_UP)
Processor = Thread->IdealProcessor;
if ((Thread->Affinity & (1 << Processor)) == 0) {
Processor = Thread->NextProcessor;
if ((Thread->Affinity & (1 << Processor)) == 0) {
FindFirstSetLeftMember(Thread->Affinity, &Processor);
}
}
Thread->NextProcessor = (CCHAR)Processor;
Prcb = KiProcessorBlock[Processor];
#endif
if (Prcb->NextThread != NULL) {
Thread1 = Prcb->NextThread;
if (ThreadPriority > Thread1->Priority) {
Thread1->Preempted = TRUE;
Prcb->NextThread = Thread;
Thread->State = Standby;
KiReadyThread(Thread1);
KiIncrementSwitchCounter(PreemptLast);
return;
}
} else {
Thread1 = Prcb->CurrentThread;
if (ThreadPriority > Thread1->Priority) {
Thread1->Preempted = TRUE;
Prcb->NextThread = Thread;
Thread->State = Standby;
KiRequestDispatchInterrupt(Thread->NextProcessor);
KiIncrementSwitchCounter(PreemptLast);
return;
}
}
}
}
//
// No thread can be preempted. Insert the thread in the dispatcher
// queue selected by its priority. If the thread was preempted and
// runs at a realtime priority level, then insert the thread at the
// front of the queue. Else insert the thread at the tail of the queue.
//
Thread->State = Ready;
if (Preempted != FALSE) {
InsertHeadList(&KiDispatcherReadyListHead[ThreadPriority],
&Thread->WaitListEntry);
} else {
InsertTailList(&KiDispatcherReadyListHead[ThreadPriority],
&Thread->WaitListEntry);
}
SetMember(ThreadPriority, KiReadySummary);
return;
}
PRKTHREAD
FASTCALL
KiSelectNextThread (
IN PRKTHREAD Thread
)
/*++
Routine Description:
This function selects the next thread to run on the processor that the
specified thread is running on. If a thread cannot be found, then the
idle thread is selected.
Arguments:
Thread - Supplies a pointer to a dispatcher object of type thread.
Return Value:
The address of the selected thread object.
--*/
{
PRKPRCB Prcb;
ULONG Processor;
PRKTHREAD Thread1;
//
// Get the processor number and the address of the processor control block.
//
#if !defined(NT_UP)
Processor = Thread->NextProcessor;
Prcb = KiProcessorBlock[Processor];
#else
Prcb = KiProcessorBlock[0];
#endif
//
// If a thread has already been selected to run on the specified processor,
// then return that thread as the selected thread.
//
if ((Thread1 = Prcb->NextThread) != NULL) {
Prcb->NextThread = (PKTHREAD)NULL;
} else {
//
// Attempt to find a ready thread to run.
//
#if !defined(NT_UP)
Thread1 = KiFindReadyThread(Processor, 0);
#else
Thread1 = KiFindReadyThread(0, 0);
#endif
//
// If a thread was not found, then select the idle thread and
// set the processor member in the idle summary.
//
if (Thread1 == NULL) {
KiIncrementSwitchCounter(SwitchToIdle);
Thread1 = Prcb->IdleThread;
#if !defined(NT_UP)
SetMember(Processor, KiIdleSummary);
#else
KiIdleSummary = 1;
#endif
}
}
//
// Return address of selected thread object.
//
return Thread1;
}
VOID
KiCalibrateTimeAdjustment (
PADJUST_INTERRUPT_TIME_CONTEXT Adjust
)
/*++
Routine Description:
This function calibrates the adjustment of time on all processors.
Arguments:
Adjust - Supplies the operation context.
Return Value:
None.
--*/
{
ULONG cl;
ULONG divisor;
BOOLEAN Enable;
LARGE_INTEGER InterruptTime;
ULARGE_INTEGER li;
PERFINFO_PO_CALIBRATED_PERFCOUNTER LogEntry;
LARGE_INTEGER NewTickCount;
ULONG NewTickOffset;
LARGE_INTEGER PerfCount;
LARGE_INTEGER PerfFreq;
LARGE_INTEGER SetTime;
//
// As each processor arrives, decrement the remaining processor count. If
// this is the last processor to arrive, then compute the time change, and
// signal all processor when to apply the performance counter change.
//
if (InterlockedDecrement((PLONG)&Adjust->KiNumber)) {
Enable = KeDisableInterrupts();
//
// It is possible to deadlock if one or more of the other processors
// receives and processes a freeze request while this processor has
// interrupts disabled. Poll for a freeze request until all processors
// are known to be in this code.
//
do {
KiPollFreezeExecution();
} while (Adjust->KiNumber != (ULONG)-1);
//
// Wait to perform the time set.
//
while (Adjust->Barrier);
} else {
//
// Set timer expiration dpc to scan the timer queues once for any
// expired timers.
//
KeRemoveQueueDpc(&KiTimerExpireDpc);
KeInsertQueueDpc(&KiTimerExpireDpc,
ULongToPtr(KiQueryLowTickCount() - TIMER_TABLE_SIZE),
NULL);
//
// Disable interrupts and indicate that this processor is now
// in final portion of this code.
//
Enable = KeDisableInterrupts();
InterlockedDecrement((PLONG) &Adjust->KiNumber);
//
// Adjust Interrupt Time.
//
InterruptTime.QuadPart = KeQueryInterruptTime() + Adjust->Adjustment;
SetTime.QuadPart = Adjust->Adjustment;
//
// Get the current times
//
PerfCount = KeQueryPerformanceCounter(&PerfFreq);
//
// Compute performance counter for current time.
//
// Multiply SetTime * PerfCount and obtain 96-bit result in cl,
// li.LowPart, li.HighPart. Then divide the 96-bit result by
// 10,000,000 to get new performance counter value.
//
li.QuadPart = RtlEnlargedUnsignedMultiply((ULONG)SetTime.LowPart,
(ULONG)PerfFreq.LowPart).QuadPart;
cl = li.LowPart;
li.QuadPart =
li.HighPart + RtlEnlargedUnsignedMultiply((ULONG)SetTime.LowPart,
(ULONG)PerfFreq.HighPart).QuadPart;
li.QuadPart =
li.QuadPart + RtlEnlargedUnsignedMultiply((ULONG)SetTime.HighPart,
(ULONG)PerfFreq.LowPart).QuadPart;
li.HighPart = li.HighPart + SetTime.HighPart * PerfFreq.HighPart;
divisor = 10000000;
Adjust->NewCount.HighPart = RtlEnlargedUnsignedDivide(li,
divisor,
&li.HighPart);
li.LowPart = cl;
Adjust->NewCount.LowPart = RtlEnlargedUnsignedDivide(li,
divisor,
NULL);
Adjust->NewCount.QuadPart += PerfCount.QuadPart;
//
// Compute tick count and tick offset for current interrupt time.
//
NewTickCount = RtlExtendedLargeIntegerDivide(InterruptTime,
KeMaximumIncrement,
&NewTickOffset);
//
// Apply changes to interrupt time, tick count, tick offset, and the
// performance counter.
//
KiTickOffset = KeMaximumIncrement - NewTickOffset;
KeInterruptTimeBias += Adjust->Adjustment;
SharedUserData->TickCount.High2Time = NewTickCount.HighPart;
#if defined(_WIN64)
SharedUserData->TickCountQuad = NewTickCount.QuadPart;
#else
SharedUserData->TickCount.LowPart = NewTickCount.LowPart;
SharedUserData->TickCount.High1Time = NewTickCount.HighPart;
#endif
//
// N.B. There is no global tick count variable on AMD64.
//
#if defined(_X86_)
KeTickCount.High2Time = NewTickCount.HighPart;
KeTickCount.LowPart = NewTickCount.LowPart;
KeTickCount.High1Time = NewTickCount.HighPart;
#endif
#if defined(_AMD64_)
SharedUserData->InterruptTime.High2Time = InterruptTime.HighPart;
*((volatile ULONG64 *)&SharedUserData->InterruptTime) = InterruptTime.QuadPart;
#else
SharedUserData->InterruptTime.High2Time = InterruptTime.HighPart;
SharedUserData->InterruptTime.LowPart = InterruptTime.LowPart;
SharedUserData->InterruptTime.High1Time = InterruptTime.HighPart;
#endif
//
// Apply the performance counter change.
//
Adjust->Barrier = 0;
}
KeGetCurrentPrcb()->TickOffset = KiTickOffset;
#if defined(_AMD64_)
KeGetCurrentPrcb()->MasterOffset = KiTickOffset;
#endif
HalCalibratePerformanceCounter((LONG volatile *)&Adjust->HalNumber,
(ULONGLONG)Adjust->NewCount.QuadPart);
//
// Log an event that the performance counter has been calibrated
// properly and indicate the new performance counter value.
//
if (PERFINFO_IS_GROUP_ON(PERF_POWER)) {
LogEntry.PerformanceCounter = KeQueryPerformanceCounter(NULL);
PerfInfoLogBytes(PERFINFO_LOG_TYPE_PO_CALIBRATED_PERFCOUNTER,
&LogEntry,
sizeof(LogEntry));
}
KeEnableInterrupts(Enable);
return;
}
VOID
KeInitializeProcess (
IN PRKPROCESS Process,
IN KPRIORITY BasePriority,
IN KAFFINITY Affinity,
IN ULONG_PTR DirectoryTableBase[2],
IN BOOLEAN Enable
)
/*++
Routine Description:
This function initializes a kernel process object. The base priority,
affinity, and page frame numbers for the process page table directory
and hyper space are stored in the process object.
N.B. It is assumed that the process object is zeroed.
Arguments:
Process - Supplies a pointer to a dispatcher object of type process.
BasePriority - Supplies the base priority of the process.
Affinity - Supplies the set of processors on which children threads
of the process can execute.
DirectoryTableBase - Supplies a pointer to an array whose fist element
is the value that is to be loaded into the Directory Table Base
register when a child thread is dispatched for execution and whose
second element contains the page table entry that maps hyper space.
Enable - Supplies a boolean value that determines the default
handling of data alignment exceptions for child threads. A value
of TRUE causes all data alignment exceptions to be automatically
handled by the kernel. A value of FALSE causes all data alignment
exceptions to be actually raised as exceptions.
Return Value:
None.
--*/
{
//
// Initialize the standard dispatcher object header and set the initial
// signal state of the process object.
//
Process->Header.Type = ProcessObject;
Process->Header.Size = sizeof(KPROCESS) / sizeof(LONG);
InitializeListHead(&Process->Header.WaitListHead);
//
// Initialize the base priority, affinity, directory table base values,
// autoalignment, and stack count.
//
// N.B. The distinguished value MAXSHORT is used to signify that no
// threads have been created for the process.
//
Process->BasePriority = (SCHAR)BasePriority;
Process->Affinity = Affinity;
Process->AutoAlignment = Enable;
Process->DirectoryTableBase[0] = DirectoryTableBase[0];
Process->DirectoryTableBase[1] = DirectoryTableBase[1];
Process->StackCount = MAXSHORT;
//
// Initialize the stack count, profile listhead, ready queue list head,
// accumulated runtime, process quantum, thread quantum, and thread list
// head.
//
InitializeListHead(&Process->ProfileListHead);
InitializeListHead(&Process->ReadyListHead);
InitializeListHead(&Process->ThreadListHead);
Process->ThreadQuantum = THREAD_QUANTUM;
//
// Initialize the process state and set the thread processor selection
// seed.
//
Process->State = ProcessInMemory;
Process->ThreadSeed = (UCHAR)KiQueryLowTickCount();
//
// Initialize IopmBase and Iopl flag for this process (i386 only)
//
#ifdef i386
Process->IopmOffset = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
#endif
return;
}
NTSTATUS
KeWaitForSingleObject (
IN PVOID Object,
IN KWAIT_REASON WaitReason,
IN KPROCESSOR_MODE WaitMode,
IN BOOLEAN Alertable,
IN PLARGE_INTEGER Timeout OPTIONAL
)
/*++
Routine Description:
This function waits until the specified object attains a state of
Signaled. An optional timeout can also be specified. If a timeout
is not specified, then the wait will not be satisfied until the object
attains a state of Signaled. If a timeout is specified, and the object
has not attained a state of Signaled when the timeout expires, then
the wait is automatically satisfied. If an explicit timeout value of
zero is specified, then no wait will occur if the wait cannot be satisfied
immediately. The wait can also be specified as alertable.
Arguments:
Object - Supplies a pointer to a dispatcher object.
WaitReason - Supplies the reason for the wait.
WaitMode - Supplies the processor mode in which the wait is to occur.
Alertable - Supplies a boolean value that specifies whether the wait is
alertable.
Timeout - Supplies a pointer to an optional absolute of relative time over
which the wait is to occur.
Return Value:
The wait completion status. A value of STATUS_TIMEOUT is returned if a
timeout occurred. A value of STATUS_SUCCESS is returned if the specified
object satisfied the wait. A value of STATUS_ALERTED is returned if the
wait was aborted to deliver an alert to the current thread. A value of
STATUS_USER_APC is returned if the wait was aborted to deliver a user
APC to the current thread.
--*/
{
LARGE_INTEGER DueTime;
LARGE_INTEGER NewTime;
PRKTHREAD NextThread;
PKMUTANT Objectx;
PLARGE_INTEGER OriginalTime;
PRKQUEUE Queue;
PRKTHREAD Thread;
PRKTIMER Timer;
PKWAIT_BLOCK WaitBlock;
NTSTATUS WaitStatus;
PKWAIT_BLOCK WaitTimer;
//
// Collect call data.
//
#if defined(_COLLECT_WAIT_SINGLE_CALLDATA_)
RECORD_CALL_DATA(&KiWaitSingleCallData);
#endif
//
// If the dispatcher database lock is not already held, then set the wait
// IRQL and lock the dispatcher database. Else set boolean wait variable
// to FALSE.
//
Thread = KeGetCurrentThread();
if (Thread->WaitNext) {
Thread->WaitNext = FALSE;
} else {
KiLockDispatcherDatabase(&Thread->WaitIrql);
}
//
// Start of wait loop.
//
// Note this loop is repeated if a kernel APC is delivered in the middle
// of the wait or a kernel APC is pending on the first attempt through
// the loop.
//
OriginalTime = Timeout;
WaitBlock = &Thread->WaitBlock[0];
do {
//
// Test to determine if a kernel APC is pending.
//
// If a kernel APC is pending and the previous IRQL was less than
// APC_LEVEL, then a kernel APC was queued by another processor just
// after IRQL was raised to DISPATCH_LEVEL, but before the dispatcher
// database was locked.
//
// N.B. that this can only happen in a multiprocessor system.
//
if (Thread->ApcState.KernelApcPending && (Thread->WaitIrql < APC_LEVEL)) {
//
// Unlock the dispatcher database and lower IRQL to its previous
// value. An APC interrupt will immediately occur which will result
// in the delivery of the kernel APC if possible.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
} else {
//
// Test if the wait can be immediately satisfied.
//
Objectx = (PKMUTANT)Object;
Thread->WaitStatus = (NTSTATUS)0;
ASSERT(Objectx->Header.Type != QueueObject);
//
// If the object is a mutant object and the mutant object has been
// recursively acquired MINLONG times, then raise an exception.
// Otherwise if the signal state of the mutant object is greater
// than zero, or the current thread is the owner of the mutant
// object, then satisfy the wait.
//
if (Objectx->Header.Type == MutantObject) {
if ((Objectx->Header.SignalState > 0) ||
(Thread == Objectx->OwnerThread)) {
if (Objectx->Header.SignalState != MINLONG) {
KiWaitSatisfyMutant(Objectx, Thread);
WaitStatus = (NTSTATUS)(Thread->WaitStatus);
goto NoWait;
} else {
KiUnlockDispatcherDatabase(Thread->WaitIrql);
ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
}
}
//
// If the signal state is greater than zero, then satisfy the wait.
//
} else if (Objectx->Header.SignalState > 0) {
KiWaitSatisfyOther(Objectx);
WaitStatus = (NTSTATUS)(0);
goto NoWait;
}
//
// Construct a wait block for the object.
//
Thread->WaitBlockList = WaitBlock;
WaitBlock->Object = Object;
WaitBlock->WaitKey = (CSHORT)(STATUS_SUCCESS);
WaitBlock->WaitType = WaitAny;
//
// Test for alert pending.
//
TestForAlertPending(Alertable);
//
// The wait cannot be satisifed immediately. Check to determine if
// a timeout value is specified.
//
if (ARGUMENT_PRESENT(Timeout)) {
//
// If the timeout value is zero, then return immediately without
// waiting.
//
if (!(Timeout->LowPart | Timeout->HighPart)) {
WaitStatus = (NTSTATUS)(STATUS_TIMEOUT);
goto NoWait;
}
//
// Initialize a wait block for the thread specific timer, insert
// wait block in timer wait list, insert the timer in the timer
// tree.
//
// N.B. The timer wait block is initialized when the respective
// thread is initialized. Thus the constant fields are not
// reinitialized. These include the wait object, wait key,
// wait type, and the wait list entry link pointers.
//
Timer = &Thread->Timer;
WaitTimer = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
WaitBlock->NextWaitBlock = WaitTimer;
Timer->Header.WaitListHead.Flink = &WaitTimer->WaitListEntry;
Timer->Header.WaitListHead.Blink = &WaitTimer->WaitListEntry;
WaitTimer->NextWaitBlock = WaitBlock;
if (KiInsertTreeTimer(Timer, *Timeout) == FALSE) {
WaitStatus = (NTSTATUS)STATUS_TIMEOUT;
goto NoWait;
}
DueTime.QuadPart = Timer->DueTime.QuadPart;
} else {
WaitBlock->NextWaitBlock = WaitBlock;
}
//
// Insert wait block in object wait list.
//
InsertTailList(&Objectx->Header.WaitListHead, &WaitBlock->WaitListEntry);
//
// If the current thread is processing a queue entry, then attempt
// to activate another thread that is blocked on the queue object.
//
Queue = Thread->Queue;
if (Queue != NULL) {
KiActivateWaiterQueue(Queue);
}
//
// Set the thread wait parameters, set the thread dispatcher state
// to Waiting, and insert the thread in the wait list.
//
Thread->Alertable = Alertable;
Thread->WaitMode = WaitMode;
Thread->WaitReason = (UCHAR)WaitReason;
Thread->WaitTime= KiQueryLowTickCount();
Thread->State = Waiting;
KiInsertWaitList(WaitMode, Thread);
//
// Switch context to selected thread.
//
// Control is returned at the original IRQL.
//
ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
WaitStatus = (NTSTATUS)KiSwapThread();
//
// If the thread was not awakened to deliver a kernel mode APC,
// then return wait status.
//
if (WaitStatus != STATUS_KERNEL_APC) {
return WaitStatus;
}
if (ARGUMENT_PRESENT(Timeout)) {
//
// Reduce the amount of time remaining before timeout occurs.
//
Timeout = KiComputeWaitInterval(OriginalTime,
&DueTime,
&NewTime);
}
}
//
// Raise IRQL to DISPATCH_LEVEL and lock the dispatcher database.
//
KiLockDispatcherDatabase(&Thread->WaitIrql);
} while (TRUE);
//
// The thread is alerted or a user APC should be delivered. Unlock the
// dispatcher database, lower IRQL to its previous value, and return
// the wait status.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
return WaitStatus;
//
// The wait has been satisfied without actually waiting.
//
// If the thread priority that is less than time critical, then reduce
// the thread quantum. If a quantum end occurs, then reduce the thread
// priority.
//
NoWait:
KiAdjustQuantumThread(Thread);
//
// Unlock the dispatcher database, lower IRQL to its previous value, and
// return the wait status.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
return WaitStatus;
}
NTSTATUS
KeWaitForMultipleObjects (
IN ULONG Count,
IN PVOID Object[],
IN WAIT_TYPE WaitType,
IN KWAIT_REASON WaitReason,
IN KPROCESSOR_MODE WaitMode,
IN BOOLEAN Alertable,
IN PLARGE_INTEGER Timeout OPTIONAL,
IN PKWAIT_BLOCK WaitBlockArray OPTIONAL
)
/*++
Routine Description:
This function waits until the specified objects attain a state of
Signaled. The wait can be specified to wait until all of the objects
attain a state of Signaled or until one of the objects attains a state
of Signaled. An optional timeout can also be specified. If a timeout
is not specified, then the wait will not be satisfied until the objects
attain a state of Signaled. If a timeout is specified, and the objects
have not attained a state of Signaled when the timeout expires, then
the wait is automatically satisfied. If an explicit timeout value of
zero is specified, then no wait will occur if the wait cannot be satisfied
immediately. The wait can also be specified as alertable.
Arguments:
Count - Supplies a count of the number of objects that are to be waited
on.
Object[] - Supplies an array of pointers to dispatcher objects.
WaitType - Supplies the type of wait to perform (WaitAll, WaitAny).
WaitReason - Supplies the reason for the wait.
WaitMode - Supplies the processor mode in which the wait is to occur.
Alertable - Supplies a boolean value that specifies whether the wait is
alertable.
Timeout - Supplies a pointer to an optional absolute of relative time over
which the wait is to occur.
WaitBlockArray - Supplies an optional pointer to an array of wait blocks
that are to used to describe the wait operation.
Return Value:
The wait completion status. A value of STATUS_TIMEOUT is returned if a
timeout occurred. The index of the object (zero based) in the object
pointer array is returned if an object satisfied the wait. A value of
STATUS_ALERTED is returned if the wait was aborted to deliver an alert
to the current thread. A value of STATUS_USER_APC is returned if the
wait was aborted to deliver a user APC to the current thread.
--*/
{
LARGE_INTEGER DueTime;
ULONG Index;
LARGE_INTEGER NewTime;
PRKTHREAD NextThread;
PKMUTANT Objectx;
PLARGE_INTEGER OriginalTime;
PRKQUEUE Queue;
PRKTHREAD Thread;
PRKTIMER Timer;
PRKWAIT_BLOCK WaitBlock;
BOOLEAN WaitSatisfied;
NTSTATUS WaitStatus;
PKWAIT_BLOCK WaitTimer;
//
// If the dispatcher database lock is not already held, then set the wait
// IRQL and lock the dispatcher database. Else set boolean wait variable
// to FALSE.
//
Thread = KeGetCurrentThread();
if (Thread->WaitNext) {
Thread->WaitNext = FALSE;
} else {
KiLockDispatcherDatabase(&Thread->WaitIrql);
}
//
// If a wait block array has been specified, then the maximum number of
// objects that can be waited on is specified by MAXIMUM_WAIT_OBJECTS.
// Otherwise the builtin wait blocks in the thread object are used and
// the maximum number of objects that can be waited on is specified by
// THREAD_WAIT_OBJECTS. If the specified number of objects is not within
// limits, then bug check.
//
if (ARGUMENT_PRESENT(WaitBlockArray)) {
if (Count > MAXIMUM_WAIT_OBJECTS) {
KeBugCheck(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
}
} else {
if (Count > THREAD_WAIT_OBJECTS) {
KeBugCheck(MAXIMUM_WAIT_OBJECTS_EXCEEDED);
}
WaitBlockArray = &Thread->WaitBlock[0];
}
//
// Start of wait loop.
//
// Note this loop is repeated if a kernel APC is delivered in the middle
// of the wait or a kernel APC is pending on the first attempt through
// the loop.
//
OriginalTime = Timeout;
do {
//
// Set address of wait block list in thread object.
//
Thread->WaitBlockList = WaitBlockArray;
//
// Test to determine if a kernel APC is pending.
//
// If a kernel APC is pending and the previous IRQL was less than
// APC_LEVEL, then a kernel APC was queued by another processor just
// after IRQL was raised to DISPATCH_LEVEL, but before the dispatcher
// database was locked.
//
// N.B. that this can only happen in a multiprocessor system.
//
if (Thread->ApcState.KernelApcPending && (Thread->WaitIrql < APC_LEVEL)) {
//
// Unlock the dispatcher database and lower IRQL to its previous
// value. An APC interrupt will immediately occur which will result
// in the delivery of the kernel APC if possible.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
} else {
//
// Construct wait blocks and check to determine if the wait is
// already satisfied. If the wait is satisfied, then perform
// wait completion and return. Else put current thread in a wait
// state if an explicit timeout value of zero is not specified.
//
Thread->WaitStatus = (NTSTATUS)0;
WaitSatisfied = TRUE;
for (Index = 0; Index < Count; Index += 1) {
//
// Test if wait can be satisfied immediately.
//
Objectx = (PKMUTANT)Object[Index];
ASSERT(Objectx->Header.Type != QueueObject);
if (WaitType == WaitAny) {
//
// If the object is a mutant object and the mutant object
// has been recursively acquired MINLONG times, then raise
// an exception. Otherwise if the signal state of the mutant
// object is greater than zero, or the current thread is
// the owner of the mutant object, then satisfy the wait.
//
if (Objectx->Header.Type == MutantObject) {
if ((Objectx->Header.SignalState > 0) ||
(Thread == Objectx->OwnerThread)) {
if (Objectx->Header.SignalState != MINLONG) {
KiWaitSatisfyMutant(Objectx, Thread);
WaitStatus = (NTSTATUS)(Index | Thread->WaitStatus);
goto NoWait;
} else {
KiUnlockDispatcherDatabase(Thread->WaitIrql);
ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
}
}
//
// If the signal state is greater than zero, then satisfy
// the wait.
//
} else if (Objectx->Header.SignalState > 0) {
KiWaitSatisfyOther(Objectx);
WaitStatus = (NTSTATUS)(Index);
goto NoWait;
}
} else {
//
// If the object is a mutant object and the mutant object
// has been recursively acquired MAXLONG times, then raise
// an exception. Otherwise if the signal state of the mutant
// object is less than or equal to zero and the current
// thread is not the owner of the mutant object, then the
// wait cannot be satisfied.
//
if (Objectx->Header.Type == MutantObject) {
if ((Thread == Objectx->OwnerThread) &&
(Objectx->Header.SignalState == MINLONG)) {
KiUnlockDispatcherDatabase(Thread->WaitIrql);
ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
} else if ((Objectx->Header.SignalState <= 0) &&
(Thread != Objectx->OwnerThread)) {
WaitSatisfied = FALSE;
}
//
// If the signal state is less than or equal to zero, then
// the wait cannot be satisfied.
//
} else if (Objectx->Header.SignalState <= 0) {
WaitSatisfied = FALSE;
}
}
//
// Construct wait block for the current object.
//
WaitBlock = &WaitBlockArray[Index];
WaitBlock->Object = (PVOID)Objectx;
WaitBlock->WaitKey = (CSHORT)(Index);
WaitBlock->WaitType = (USHORT)WaitType;
WaitBlock->Thread = Thread;
WaitBlock->NextWaitBlock = &WaitBlockArray[Index + 1];
}
//
// If the wait type is wait all, then check to determine if the
// wait can be satisfied immediately.
//
if ((WaitType == WaitAll) && (WaitSatisfied)) {
WaitBlock->NextWaitBlock = &WaitBlockArray[0];
KiWaitSatisfyAll(WaitBlock);
WaitStatus = (NTSTATUS)Thread->WaitStatus;
goto NoWait;
}
//
// Test for alert pending.
//
TestForAlertPending(Alertable);
//
// The wait cannot be satisifed immediately. Check to determine if
// a timeout value is specified.
//
if (ARGUMENT_PRESENT(Timeout)) {
//
// If the timeout value is zero, then return immediately without
// waiting.
//
if (!(Timeout->LowPart | Timeout->HighPart)) {
WaitStatus = (NTSTATUS)(STATUS_TIMEOUT);
goto NoWait;
}
//
// Initialize a wait block for the thread specific timer,
// initialize timer wait list head, insert the timer in the
// timer tree, and increment the number of wait objects.
//
// N.B. The timer wait block is initialized when the respective
// thread is initialized. Thus the constant fields are not
// reinitialized. These include the wait object, wait key,
// wait type, and the wait list entry link pointers.
//
WaitTimer = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
WaitBlock->NextWaitBlock = WaitTimer;
WaitBlock = WaitTimer;
Timer = &Thread->Timer;
InitializeListHead(&Timer->Header.WaitListHead);
if (KiInsertTreeTimer(Timer, *Timeout) == FALSE) {
WaitStatus = (NTSTATUS)STATUS_TIMEOUT;
goto NoWait;
}
DueTime.QuadPart = Timer->DueTime.QuadPart;
}
//
// Close up the circular list of wait control blocks.
//
WaitBlock->NextWaitBlock = &WaitBlockArray[0];
//
// Insert wait blocks in object wait lists.
//
WaitBlock = &WaitBlockArray[0];
do {
Objectx = (PKMUTANT)WaitBlock->Object;
InsertTailList(&Objectx->Header.WaitListHead, &WaitBlock->WaitListEntry);
WaitBlock = WaitBlock->NextWaitBlock;
} while (WaitBlock != &WaitBlockArray[0]);
//
// If the current thread is processing a queue entry, then attempt
// to activate another thread that is blocked on the queue object.
//
Queue = Thread->Queue;
if (Queue != NULL) {
KiActivateWaiterQueue(Queue);
}
//
// Set the thread wait parameters, set the thread dispatcher state
// to Waiting, and insert the thread in the wait list.
//
Thread->Alertable = Alertable;
Thread->WaitMode = WaitMode;
Thread->WaitReason = (UCHAR)WaitReason;
Thread->WaitTime= KiQueryLowTickCount();
Thread->State = Waiting;
KiInsertWaitList(WaitMode, Thread);
//
// Switch context to selected thread.
//
// Control is returned at the original IRQL.
//
ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
WaitStatus = (NTSTATUS)KiSwapThread();
//
// If the thread was not awakened to deliver a kernel mode APC,
// then the wait status.
//
if (WaitStatus != STATUS_KERNEL_APC) {
return WaitStatus;
}
if (ARGUMENT_PRESENT(Timeout)) {
//
// Reduce the amount of time remaining before timeout occurs.
//
Timeout = KiComputeWaitInterval(OriginalTime,
&DueTime,
&NewTime);
}
}
//
// Raise IRQL to DISPATCH_LEVEL and lock the dispatcher database.
//
KiLockDispatcherDatabase(&Thread->WaitIrql);
} while (TRUE);
//
// The thread is alerted or a user APC should be delivered. Unlock the
// dispatcher database, lower IRQL to its previous value, and return
// the wait status.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
return WaitStatus;
//
// The wait has been satisfied without actually waiting.
//
// If the thread priority that is less than time critical, then reduce
// the thread quantum. If a quantum end occurs, then reduce the thread
// priority.
//
NoWait:
KiAdjustQuantumThread(Thread);
//
// Unlock the dispatcher database, lower IRQL to its previous value, and
// return the wait status.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
return WaitStatus;
}
NTSTATUS
KeDelayExecutionThread (
IN KPROCESSOR_MODE WaitMode,
IN BOOLEAN Alertable,
IN PLARGE_INTEGER Interval
)
/*++
Routine Description:
This function delays the execution of the current thread for the specified
interval of time.
Arguments:
WaitMode - Supplies the processor mode in which the delay is to occur.
Alertable - Supplies a boolean value that specifies whether the delay
is alertable.
Interval - Supplies a pointer to the absolute or relative time over which
the delay is to occur.
Return Value:
The wait completion status. A value of STATUS_SUCCESS is returned if
the delay occurred. A value of STATUS_ALERTED is returned if the wait
was aborted to deliver an alert to the current thread. A value of
STATUS_USER_APC is returned if the wait was aborted to deliver a user
APC to the current thread.
--*/
{
LARGE_INTEGER DueTime;
LARGE_INTEGER NewTime;
PLARGE_INTEGER OriginalTime;
PKPRCB Prcb;
KPRIORITY Priority;
PRKQUEUE Queue;
PRKTHREAD Thread;
PRKTIMER Timer;
PKWAIT_BLOCK WaitBlock;
NTSTATUS WaitStatus;
//
// If the dispatcher database lock is not already held, then set the wait
// IRQL and lock the dispatcher database. Else set boolean wait variable
// to FALSE.
//
Thread = KeGetCurrentThread();
if (Thread->WaitNext) {
Thread->WaitNext = FALSE;
} else {
KiLockDispatcherDatabase(&Thread->WaitIrql);
}
//
// Start of delay loop.
//
// Note this loop is repeated if a kernel APC is delivered in the middle
// of the delay or a kernel APC is pending on the first attempt through
// the loop.
//
OriginalTime = Interval;
WaitBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
do {
//
// Test to determine if a kernel APC is pending.
//
// If a kernel APC is pending and the previous IRQL was less than
// APC_LEVEL, then a kernel APC was queued by another processor just
// after IRQL was raised to DISPATCH_LEVEL, but before the dispatcher
// database was locked.
//
// N.B. that this can only happen in a multiprocessor system.
//
if (Thread->ApcState.KernelApcPending && (Thread->WaitIrql < APC_LEVEL)) {
//
// Unlock the dispatcher database and lower IRQL to its previous
// value. An APC interrupt will immediately occur which will result
// in the delivery of the kernel APC if possible.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
} else {
//
// Test for alert pending.
//
TestForAlertPending(Alertable);
//
// Initialize wait block, insert wait block in timer wait list,
// insert timer in timer queue, put thread in wait state, select
// next thread to execute, and context switch to next thread.
//
// N.B. The timer wait block is initialized when the respective
// thread is initialized. Thus the constant fields are not
// reinitialized. These include the wait object, wait key,
// wait type, and the wait list entry link pointers.
//
Thread->WaitBlockList = WaitBlock;
Thread->WaitStatus = (NTSTATUS)0;
Timer = &Thread->Timer;
WaitBlock->NextWaitBlock = WaitBlock;
Timer->Header.WaitListHead.Flink = &WaitBlock->WaitListEntry;
Timer->Header.WaitListHead.Blink = &WaitBlock->WaitListEntry;
//
// If the timer is inserted in the timer tree, then place the
// current thread in a wait state. Otherwise, attempt to force
// the current thread to yield the processor to another thread.
//
if (KiInsertTreeTimer(Timer, *Interval) == FALSE) {
//
// If the thread is not a realtime thread, then drop the
// thread priority to the base priority.
//
Prcb = KeGetCurrentPrcb();
Priority = Thread->Priority;
if (Priority < LOW_REALTIME_PRIORITY) {
if (Priority != Thread->BasePriority) {
Thread->PriorityDecrement = 0;
KiSetPriorityThread(Thread, Thread->BasePriority);
}
}
//
// If a new thread has not been selected, the attempt to round
// robin the thread with other threads at the same priority.
//
if (Prcb->NextThread == NULL) {
Prcb->NextThread = KiFindReadyThread(Thread->NextProcessor,
Thread->Priority);
}
//
// If a new thread has been selected for execution, then
// switch immediately to the selected thread.
//
if (Prcb->NextThread != NULL) {
//
// Give the current thread a new qunatum and switch
// context to selected thread.
//
// N.B. Control is returned at the original IRQL.
//
Thread->Preempted = FALSE;
Thread->Quantum = Thread->ApcState.Process->ThreadQuantum;
ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
KiReadyThread(Thread);
WaitStatus = (NTSTATUS)KiSwapThread();
goto WaitComplete;
} else {
WaitStatus = (NTSTATUS)STATUS_SUCCESS;
break;
}
}
DueTime.QuadPart = Timer->DueTime.QuadPart;
//
// If the current thread is processing a queue entry, then attempt
// to activate another thread that is blocked on the queue object.
//
Queue = Thread->Queue;
if (Queue != NULL) {
KiActivateWaiterQueue(Queue);
}
//
// Set the thread wait parameters, set the thread dispatcher state
// to Waiting, and insert the thread in the wait list.
//
Thread->Alertable = Alertable;
Thread->WaitMode = WaitMode;
Thread->WaitReason = DelayExecution;
Thread->WaitTime= KiQueryLowTickCount();
Thread->State = Waiting;
KiInsertWaitList(WaitMode, Thread);
//
// Switch context to selected thread.
//
// N.B. Control is returned at the original IRQL.
//
ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
WaitStatus = (NTSTATUS)KiSwapThread();
//
// If the thread was not awakened to deliver a kernel mode APC,
// then return the wait status.
//
WaitComplete:
if (WaitStatus != STATUS_KERNEL_APC) {
if (WaitStatus == STATUS_TIMEOUT) {
WaitStatus = STATUS_SUCCESS;
}
return WaitStatus;
}
//
// Reduce the time remaining before the time delay expires.
//
Interval = KiComputeWaitInterval(OriginalTime,
&DueTime,
&NewTime);
}
//
// Raise IRQL to DISPATCH_LEVEL and lock the dispatcher database.
//
KiLockDispatcherDatabase(&Thread->WaitIrql);
} while (TRUE);
//
// The thread is alerted or a user APC should be delivered. Unlock the
// dispatcher database, lower IRQL to its previous value, and return the
// wait status.
//
KiUnlockDispatcherDatabase(Thread->WaitIrql);
return WaitStatus;
}