LONG
KeResetEvent (
__inout PRKEVENT Event
)
/*++
Routine Description:
This function resets the signal state of an event object to
Not-Signaled. The previous state of the event object is returned
as the function value.
Arguments:
Event - Supplies a pointer to a dispatcher object of type event.
Return Value:
The previous signal state of the event object.
--*/
{
KIRQL OldIrql;
LONG OldState;
ASSERT_EVENT(Event);
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
//
// Raise IRQL to dispatcher level and lock dispatcher database.
//
KiLockDispatcherDatabase(&OldIrql);
//
// Capture the current signal state of event object and then reset
// the state of the event object to Not-Signaled.
//
OldState = ReadForWriteAccess(&Event->Header.SignalState);
Event->Header.SignalState = 0;
//
// Unlock the dispatcher database and lower IRQL to its previous
// value.
KiUnlockDispatcherDatabase(OldIrql);
//
// Return previous signal state of event object.
//
return OldState;
}
ULONG
BtrIsUnloading(
VOID
)
{
ULONG Unloading;
Unloading = ReadForWriteAccess(&BtrUnloading);
return Unloading;
}
VOID
BtrAcquireSpinLock(
IN PBTR_SPINLOCK Lock
)
{
ULONG Acquired;
ULONG ThreadId;
ULONG OwnerId;
ULONG Count;
ULONG SpinCount;
ThreadId = GetCurrentThreadId();
OwnerId = ReadForWriteAccess(&Lock->ThreadId);
if (OwnerId == ThreadId) {
return;
}
SpinCount = Lock->SpinCount;
while (TRUE) {
Acquired = InterlockedBitTestAndSet((volatile LONG *)&Lock->Acquired, 0);
if (Acquired != 1) {
Lock->ThreadId = ThreadId;
break;
}
Count = 0;
do {
YieldProcessor();
Acquired = ReadForWriteAccess(&Lock->Acquired);
Count += 1;
if (Count >= SpinCount) {
SwitchToThread();
break;
}
} while (Acquired != 0);
}
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfReleasePushLockExclusive (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Release a push lock that was acquired exclusive
Arguments:
PushLock - Push lock to be released
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
OldValue = ReadForWriteAccess (PushLock);
while (1) {
ASSERT (OldValue.Locked);
ASSERT (OldValue.Waiting || OldValue.Shared == 0);
if (OldValue.Waiting && !OldValue.Waking) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_LOCK + EX_PUSH_LOCK_WAKING;
ASSERT (NewValue.Waking && !NewValue.Locked);
TopValue = NewValue;
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
ExfWakePushLock (PushLock, TopValue);
break;
}
} else {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_LOCK;
ASSERT (NewValue.Waking || !NewValue.Waiting);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
break;
}
}
OldValue = NewValue;
}
}
NTKERNELAPI
BOOLEAN
FASTCALL
ExfTryAcquirePushLockShared (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Try to acquire a push lock shared without blocking
Arguments:
PushLock - Push lock to be acquired
Return Value:
BOOLEAN - TRUE, The lock was acquired, FALSE otherwise
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
OldValue = ReadForWriteAccess (PushLock);
while (1) {
//
// If the lock is already held we need to wait if its not held shared
//
if (!OldValue.Locked || (!OldValue.Waiting && OldValue.Shared > 0)) {
if (OldValue.Waiting) {
NewValue.Value = OldValue.Value + EX_PUSH_LOCK_LOCK;
} else {
NewValue.Value = (OldValue.Value + EX_PUSH_LOCK_SHARE_INC) | EX_PUSH_LOCK_LOCK;
}
ASSERT (NewValue.Locked);
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
return TRUE;
}
} else {
return FALSE;
}
OldValue = NewValue;
}
}
NTKERNELAPI
VOID
FASTCALL
ExBlockPushLock (
__inout PEX_PUSH_LOCK PushLock,
__inout PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
)
/*++
Routine Description:
Block on a push lock
Arguments:
PushLock - Push lock to block on
WaitBlock - Wait block to queue for waiting
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
//
// Mark the wait block as not waiting yet
//
WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_SPINNING;
//
// Push the wait block on the list.
//
OldValue = ReadForWriteAccess (PushLock);
while (1) {
//
// Chain the next block to us if there is one.
//
WaitBlock->Next = OldValue.Ptr;
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
WaitBlock,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
return;
}
OldValue = NewValue;
}
}
NTKERNELAPI
VOID
FASTCALL
ExfConvertPushLockExclusiveToShared (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Attempts to convert an exclusive acquire to shared. If other shared waiters
are present at the end of the waiters chain they are released.
Arguments:
PushLock - Push lock to be converted
Return Value:
None.
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
OldValue = ReadForWriteAccess (PushLock);
while (1) {
ASSERT (OldValue.Locked);
ASSERT (OldValue.Waiting || OldValue.Shared == 0);
if (!OldValue.Waiting) {
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
(PVOID) (EX_PUSH_LOCK_LOCK|EX_PUSH_LOCK_SHARE_INC),
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
return;
}
OldValue = NewValue;
} else {
return;
}
}
}
LONG
KeSetEvent (
__inout PRKEVENT Event,
__in KPRIORITY Increment,
__in BOOLEAN Wait
)
/*++
Routine Description:
This function sets the signal state of an event object to signaled
and attempts to satisfy as many waits as possible. The previous
signal state of the event object is returned as the function value.
Arguments:
Event - Supplies a pointer to a dispatcher object of type event.
Increment - Supplies the priority increment that is to be applied
if setting the event causes a Wait to be satisfied.
Wait - Supplies a boolean value that signifies whether the call to
set event will be immediately followed by a call to one of the
kernel Wait functions.
Return Value:
The previous signal state of the event object.
--*/
{
KIRQL OldIrql;
LONG OldState;
PRKTHREAD Thread;
ASSERT_EVENT(Event);
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
//
// If the event is a notification event, the event is already signaled,
// and wait is false, then there is no need to set the event.
//
if ((Event->Header.Type == EventNotificationObject) &&
(Event->Header.SignalState == 1) &&
(Wait == FALSE)) {
return 1;
}
//
// Raise IRQL to dispatcher level and lock dispatcher database.
//
KiLockDispatcherDatabase(&OldIrql);
//
// Capture the old state and set the new state to signaled.
//
// If the old state is not-signaled and the wait list is not empty,
// then satisfy as many waits as possible.
//
OldState = ReadForWriteAccess(&Event->Header.SignalState);
Event->Header.SignalState = 1;
if ((OldState == 0) &&
(IsListEmpty(&Event->Header.WaitListHead) == FALSE)) {
if (Event->Header.Type == EventNotificationObject) {
KiWaitTestWithoutSideEffects(Event, Increment);
} else {
KiWaitTestSynchronizationObject(Event, Increment);
}
}
//
// If the value of the Wait argument is TRUE, then return to the
// caller with IRQL raised and the dispatcher database locked. Else
// release the dispatcher database lock and lower IRQL to its
// previous value.
//
if (Wait != FALSE) {
Thread = KeGetCurrentThread();
Thread->WaitNext = Wait;
Thread->WaitIrql = OldIrql;
} else {
KiUnlockDispatcherDatabase(OldIrql);
}
//
// Return previous signal state of event object.
//
return OldState;
}
LONG
KePulseEvent (
__inout PRKEVENT Event,
__in KPRIORITY Increment,
__in BOOLEAN Wait
)
/*++
Routine Description:
This function atomically sets the signal state of an event object to
signaled, attempts to satisfy as many waits as possible, and then resets
the signal state of the event object to Not-Signaled. The previous signal
state of the event object is returned as the function value.
Arguments:
Event - Supplies a pointer to a dispatcher object of type event.
Increment - Supplies the priority increment that is to be applied
if setting the event causes a Wait to be satisfied.
Wait - Supplies a boolean value that signifies whether the call to
KePulseEvent will be immediately followed by a call to one of the
kernel Wait functions.
Return Value:
The previous signal state of the event object.
--*/
{
KIRQL OldIrql;
LONG OldState;
PRKTHREAD Thread;
ASSERT_EVENT(Event);
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
//
// Raise IRQL to dispatcher level and lock dispatcher database.
//
KiLockDispatcherDatabase(&OldIrql);
//
// If the current state of the event object is Not-Signaled and
// the wait queue is not empty, then set the state of the event
// to Signaled, satisfy as many Waits as possible, and then reset
// the state of the event to Not-Signaled.
//
OldState = ReadForWriteAccess(&Event->Header.SignalState);
if ((OldState == 0) &&
(IsListEmpty(&Event->Header.WaitListHead) == FALSE)) {
Event->Header.SignalState = 1;
KiWaitTest(Event, Increment);
}
Event->Header.SignalState = 0;
//
// If the value of the Wait argument is TRUE, then return to the
// caller with IRQL raised and the dispatcher database locked. Else
// release the dispatcher database lock and lower IRQL to the
// previous value.
//
if (Wait != FALSE) {
Thread = KeGetCurrentThread();
Thread->WaitIrql = OldIrql;
Thread->WaitNext = Wait;
} else {
KiUnlockDispatcherDatabase(OldIrql);
}
//
// Return previous signal state of event object.
//
return OldState;
}
VOID
ExpWorkerThread (
IN PVOID StartContext
)
{
PLIST_ENTRY Entry;
WORK_QUEUE_TYPE QueueType;
PWORK_QUEUE_ITEM WorkItem;
KPROCESSOR_MODE WaitMode;
LARGE_INTEGER TimeoutValue;
PLARGE_INTEGER Timeout;
PETHREAD Thread;
PEX_WORK_QUEUE WorkerQueue;
PWORKER_THREAD_ROUTINE WorkerRoutine;
PVOID Parameter;
EX_QUEUE_WORKER_INFO OldWorkerInfo;
EX_QUEUE_WORKER_INFO NewWorkerInfo;
ULONG CountForQueueEmpty;
//
// Set timeout value etc according to whether we are static or dynamic.
//
if (((ULONG_PTR)StartContext & DYNAMIC_WORKER_THREAD) == 0) {
//
// We are being created as a static thread. As such it will not
// terminate, so there is no point in timing out waiting for a work
// item.
//
Timeout = NULL;
}
else {
//
// This is a dynamic worker thread. It has a non-infinite timeout
// so that it can eventually terminate.
//
TimeoutValue.QuadPart = -DYNAMIC_THREAD_TIMEOUT;
Timeout = &TimeoutValue;
}
Thread = PsGetCurrentThread ();
//
// If the thread is a critical worker thread, then set the thread
// priority to the lowest realtime level. Otherwise, set the base
// thread priority to time critical.
//
QueueType = (WORK_QUEUE_TYPE)
((ULONG_PTR)StartContext & ~DYNAMIC_WORKER_THREAD);
WorkerQueue = &ExWorkerQueue[QueueType];
WaitMode = (KPROCESSOR_MODE) WorkerQueue->Info.WaitMode;
ASSERT (Thread->ExWorkerCanWaitUser == 0);
if (WaitMode == UserMode) {
Thread->ExWorkerCanWaitUser = 1;
}
#if defined(REMOTE_BOOT)
//
// In diskless NT scenarios ensure that the kernel stack of the worker
// threads will not be swapped out.
//
if (IoRemoteBootClient) {
KeSetKernelStackSwapEnable (FALSE);
}
#endif // defined(REMOTE_BOOT)
//
// Register as a worker, exiting if the queue's going down and
// there aren't any workers in the queue to hand us the shutdown
// work item if we enter the queue (we want to be able to enter a
// queue even if the queue's shutting down, in case there's a
// backlog of work items that the balance manager thread's decided
// we should be helping to process).
//
if (PO_SHUTDOWN_QUEUE == QueueType) {
CountForQueueEmpty = 1;
}
else {
CountForQueueEmpty = 0;
}
if (ExpWorkersCanSwap == FALSE) {
KeSetKernelStackSwapEnable (FALSE);
}
do {
OldWorkerInfo.QueueWorkerInfo = ReadForWriteAccess (&WorkerQueue->Info.QueueWorkerInfo);
if (OldWorkerInfo.QueueDisabled &&
OldWorkerInfo.WorkerCount <= CountForQueueEmpty) {
//
// The queue is disabled and empty so just exit.
//
KeSetKernelStackSwapEnable (TRUE);
PsTerminateSystemThread (STATUS_SYSTEM_SHUTDOWN);
}
NewWorkerInfo.QueueWorkerInfo = OldWorkerInfo.QueueWorkerInfo;
NewWorkerInfo.WorkerCount += 1;
} while (OldWorkerInfo.QueueWorkerInfo !=
InterlockedCompareExchange (&WorkerQueue->Info.QueueWorkerInfo,
NewWorkerInfo.QueueWorkerInfo,
OldWorkerInfo.QueueWorkerInfo));
//
// As of this point, we must only exit if we decrement the worker
// count without the queue disabled flag being set. (Unless we
// exit due to the shutdown work item, which also decrements the
// worker count).
//
Thread->ActiveExWorker = 1;
//
// Loop forever waiting for a work queue item, calling the processing
// routine, and then waiting for another work queue item.
//
do {
//
// Wait until something is put in the queue or until we time out.
//
// By specifying a wait mode of UserMode, the thread's kernel
// stack is swappable.
//
Entry = KeRemoveQueue (&WorkerQueue->WorkerQueue,
WaitMode,
Timeout);
if ((ULONG_PTR)Entry != STATUS_TIMEOUT) {
//
// This is a real work item, process it.
//
// Update the total number of work items processed.
//
InterlockedIncrement ((PLONG)&WorkerQueue->WorkItemsProcessed);
WorkItem = CONTAINING_RECORD(Entry, WORK_QUEUE_ITEM, List);
WorkerRoutine = WorkItem->WorkerRoutine;
Parameter = WorkItem->Parameter;
//
// Catch worker routines referencing a user mode address.
//
ASSERT ((ULONG_PTR)WorkerRoutine > MmUserProbeAddress);
//
// Execute the specified routine.
//
((PWORKER_THREAD_ROUTINE)WorkerRoutine) (Parameter);
#if DBG
if (IsListEmpty (&Thread->IrpList)) {
//
// See if a worker just returned while holding a resource
//
ExCheckIfResourceOwned ();
}
#endif
//
// Catch worker routines that forget to leave a critial/guarded
// region. In the debug case execute a breakpoint. In the free
// case zero the flag so that APCs can continue to fire to this
// thread.
//
if (Thread->Tcb.CombinedApcDisable != 0) {
DbgPrint ((char*)ExpWorkerApcDisabledMessage,
WorkerRoutine,
Parameter,
WorkItem);
ASSERT (FALSE);
Thread->Tcb.CombinedApcDisable = 0;
}
if (KeGetCurrentIrql () != PASSIVE_LEVEL) {
KeBugCheckEx (WORKER_THREAD_RETURNED_AT_BAD_IRQL,
(ULONG_PTR)WorkerRoutine,
(ULONG_PTR)KeGetCurrentIrql(),
(ULONG_PTR)Parameter,
(ULONG_PTR)WorkItem);
}
if (PS_IS_THREAD_IMPERSONATING (Thread)) {
KeBugCheckEx (IMPERSONATING_WORKER_THREAD,
(ULONG_PTR)WorkerRoutine,
(ULONG_PTR)Parameter,
(ULONG_PTR)WorkItem,
0);
}
continue;
}
//
// These things are known:
//
// - Static worker threads do not time out, so this is a dynamic
// worker thread.
//
// - This thread has been waiting for a long time with nothing
// to do.
//
if (IsListEmpty (&Thread->IrpList) == FALSE) {
//
// There is still I/O pending, can't terminate yet.
//
continue;
}
//
// Get out of the queue, if we can
//
do {
OldWorkerInfo.QueueWorkerInfo = ReadForWriteAccess (&WorkerQueue->Info.QueueWorkerInfo);
if (OldWorkerInfo.QueueDisabled) {
//
// We're exiting via the queue disable work item;
// there's no point in expiring here.
//
break;
}
NewWorkerInfo.QueueWorkerInfo = OldWorkerInfo.QueueWorkerInfo;
NewWorkerInfo.WorkerCount -= 1;
} while (OldWorkerInfo.QueueWorkerInfo
!= InterlockedCompareExchange(&WorkerQueue->Info.QueueWorkerInfo,
NewWorkerInfo.QueueWorkerInfo,
OldWorkerInfo.QueueWorkerInfo));
if (OldWorkerInfo.QueueDisabled) {
//
// We're exiting via the queue disable work item
//
continue;
}
//
// This dynamic thread can be terminated.
//
break;
} while (TRUE);
//
// Terminate this dynamic thread.
//
InterlockedDecrement ((PLONG)&WorkerQueue->DynamicThreadCount);
//
// Carefully clear this before marking the thread stack as swap enabled
// so that an incoming APC won't inadvertently disable the stack swap
// afterwards.
//
Thread->ActiveExWorker = 0;
//
// We will bugcheck if we terminate a thread with stack swapping
// disabled.
//
KeSetKernelStackSwapEnable (TRUE);
return;
}
BOOLEAN
ExRemoveHeadNBQueue (
IN PVOID Header,
OUT PULONG64 Value
)
/*++
Routine Description:
This function removes a queue entry from the head of the specified
non-blocking queue and returns the associated data value.
Arguments:
Header - Supplies an opaque pointer to a non-blocking queue header.
Value - Supplies a pointer to a variable that receives the queue
element value.
Return Value:
If an entry is removed from the specified non-blocking queue, then
TRUE is returned as the function value. Otherwise, FALSE is returned.
--*/
{
NBQUEUE_POINTER Head;
PNBQUEUE_NODE HeadNode;
NBQUEUE_POINTER Insert;
NBQUEUE_POINTER Next;
PNBQUEUE_NODE NextNode;
PNBQUEUE_HEADER QueueHead;
NBQUEUE_POINTER Tail;
PNBQUEUE_NODE TailNode;
//
// The following loop is executed until an entry can be removed from
// the specified non-blocking queue or until it can be determined that
// the queue is empty.
//
QueueHead = (PNBQUEUE_HEADER)Header;
do {
//
// Read the head queue pointer, the tail queue pointer, and the
// next queue pointer of the head queue pointer making sure the
// three pointers are coherent.
//
Head.Data = ReadForWriteAccess((volatile LONG64 *)(&QueueHead->Head.Data));
Tail.Data = *((volatile LONG64 *)(&QueueHead->Tail.Data));
HeadNode = UnpackNBQPointer(&Head);
Next.Data = *((volatile LONG64 *)(&HeadNode->Next.Data));
if (Head.Data == *((volatile LONG64 *)(&QueueHead->Head.Data))) {
//
// If the queue header node is equal to the queue tail node,
// then either the queue is empty or the tail pointer is falling
// behind. Otherwise, there is an entry in the queue that can
// be removed.
//
NextNode = UnpackNBQPointer(&Next);
TailNode = UnpackNBQPointer(&Tail);
if (HeadNode == TailNode) {
//
// If the next node of head pointer is NULL, then the queue
// is empty. Otherwise, attempt to move the tail forward.
//
if (NextNode == NULL) {
return FALSE;
} else {
PackNBQPointer(&Insert, NextNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
}
} else {
//
// There is an entry in the queue that can be removed.
//
*Value = NextNode->Value;
PackNBQPointer(&Insert, NextNode);
Insert.Count = Head.Count + 1;
if (InterlockedCompareExchange64(&QueueHead->Head.Data,
Insert.Data,
Head.Data) == Head.Data) {
break;
}
}
}
} while (TRUE);
//
// Free the node that was removed for the list by inserting the node
// in the associated SLIST.
//
InterlockedPushEntrySList(QueueHead->SlistHead,
(PSLIST_ENTRY)HeadNode);
return TRUE;
}
BOOLEAN
ExInsertTailNBQueue (
IN PVOID Header,
IN ULONG64 Value
)
/*++
Routine Description:
This function inserts the specific data value at the tail of the
specified non-blocking queue.
Arguments:
Header - Supplies an opaque pointer to a non-blocking queue header.
Value - Supplies a pointer to an opaque data value.
Return Value:
If the specified opaque data value is successfully inserted at the tail
of the specified non-blocking queue, then a value of TRUE is returned as
the function value. Otherwise, a value of FALSE is returned.
N.B. FALSE is returned if a queue node cannot be allocated from the
associated SLIST.
--*/
{
NBQUEUE_POINTER Insert;
NBQUEUE_POINTER Next;
PNBQUEUE_NODE NextNode;
PNBQUEUE_HEADER QueueHead;
PNBQUEUE_NODE QueueNode;
NBQUEUE_POINTER Tail;
PNBQUEUE_NODE TailNode;
//
// Attempt to allocate a queue node from the SLIST associated with
// the specified non-blocking queue. If a node can be allocated, then
// the node is inserted at the tail of the specified non-blocking
// queue, and TRUE is returned as the function value. Otherwise, FALSE
// is returned.
//
QueueHead = (PNBQUEUE_HEADER)Header;
QueueNode = (PNBQUEUE_NODE)InterlockedPopEntrySList(QueueHead->SlistHead);
if (QueueNode != NULL) {
//
// Initialize the queue node next pointer and value.
//
QueueNode->Next.Data = 0;
QueueNode->Value = Value;
//
// The following loop is executed until the specified entry can
// be safely inserted at the tail of the specified non-blocking
// queue.
//
do {
//
// Read the tail queue pointer and the next queue pointer of
// the tail queue pointer making sure the two pointers are
// coherent.
//
Tail.Data = ReadForWriteAccess((volatile LONG64 *)(&QueueHead->Tail.Data));
TailNode = UnpackNBQPointer(&Tail);
Next.Data = *((volatile LONG64 *)(&TailNode->Next.Data));
QueueNode->Next.Count = Tail.Count + 1;
if (Tail.Data == *((volatile LONG64 *)(&QueueHead->Tail.Data))) {
//
// If the tail is pointing to the last node in the list,
// then attempt to insert the new node at the end of the
// list. Otherwise, the tail is not pointing to the last
// node in the list and an attempt is made to move the
// tail pointer to the next node.
//
NextNode = UnpackNBQPointer(&Next);
if (NextNode == NULL) {
PackNBQPointer(&Insert, QueueNode);
Insert.Count = Next.Count + 1;
if (InterlockedCompareExchange64(&TailNode->Next.Data,
Insert.Data,
Next.Data) == Next.Data) {
break;
}
} else {
PackNBQPointer(&Insert, NextNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
}
}
} while (TRUE);
//
// Attempt to move the tail to the new tail node.
//
PackNBQPointer(&Insert, QueueNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
return TRUE;
} else {
return FALSE;
}
}
DECLSPEC_NOINLINE
VOID
KiIpiProcessRequests (
VOID
)
/*++
Routine Description:
This routine processes interprocessor requests and returns a summary
of the requests that were processed.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
PVOID *End;
ULONG64 Number;
PKPRCB Packet;
PKPRCB Prcb;
ULONG Processor;
REQUEST_SUMMARY Request;
PREQUEST_MAILBOX RequestMailbox;
PKREQUEST_PACKET RequestPacket;
LONG64 SetMember;
PKPRCB Source;
KAFFINITY SummarySet;
KAFFINITY TargetSet;
PVOID *Virtual;
//
// Loop until the sender summary is zero.
//
Prcb = KeGetCurrentPrcb();
TargetSet = ReadForWriteAccess(&Prcb->SenderSummary);
SetMember = Prcb->SetMember;
while (TargetSet != 0) {
SummarySet = TargetSet;
BitScanForward64(&Processor, SummarySet);
do {
Source = KiProcessorBlock[Processor];
RequestMailbox = &Prcb->RequestMailbox[Processor];
Request.Summary = RequestMailbox->RequestSummary;
//
// If the request type is flush multiple immediate, flush process,
// flush single, or flush all, then packet done can be signaled
// before processing the request. Otherwise, the request type must
// be a packet request, a cache invalidate, or a flush multiple
//
if (Request.IpiRequest <= IPI_FLUSH_ALL) {
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE_IMMEDIATE) {
Number = Request.Count;
Virtual = &RequestMailbox->Virtual[0];
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_FLUSH_PROCESS) {
KiFlushProcessTb();
} else if (Request.IpiRequest == IPI_FLUSH_SINGLE) {
KiFlushSingleTb((PVOID)Request.Parameter);
} else {
ASSERT(Request.IpiRequest == IPI_FLUSH_ALL);
KeFlushCurrentTb();
}
} else {
//
// If the request type is packet ready, then call the worker
// function. Otherwise, the request must be either a flush
// multiple or a cache invalidate.
//
if (Request.IpiRequest == IPI_PACKET_READY) {
Packet = Source;
if (Request.IpiSynchType != 0) {
Packet = (PKPRCB)((ULONG64)Source + 1);
}
RequestPacket = (PKREQUEST_PACKET)&RequestMailbox->RequestPacket;
(RequestPacket->WorkerRoutine)((PKIPI_CONTEXT)Packet,
RequestPacket->CurrentPacket[0],
RequestPacket->CurrentPacket[1],
RequestPacket->CurrentPacket[2]);
} else {
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE) {
Number = Request.Count;
Virtual = (PVOID *)Request.Parameter;
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_INVALIDATE_ALL) {
WritebackInvalidate();
} else {
ASSERT(FALSE);
}
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
}
}
SummarySet ^= AFFINITY_MASK(Processor);
} while (BitScanForward64(&Processor, SummarySet) != FALSE);
//
// Clear target set in sender summary.
//
TargetSet =
InterlockedExchangeAdd64((LONG64 volatile *)&Prcb->SenderSummary,
-(LONG64)TargetSet) - TargetSet;
}
#endif
return;
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfReleasePushLock (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Release a push lock that was acquired exclusively or shared
Arguments:
PushLock - Push lock to be released
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock;
OldValue = ReadForWriteAccess (PushLock);
ASSERT (OldValue.Locked);
while (1) {
if (!OldValue.Waiting) {
if (OldValue.Shared > 1) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
} else {
NewValue.Value = 0;
}
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
return;
}
OldValue = NewValue;
} else {
//
// See if we need to walk the list to find the share count
//
if (OldValue.MultipleShared) {
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK) (OldValue.Value & ~(ULONG_PTR)EX_PUSH_LOCK_PTR_BITS);
while (1) {
NextWaitBlock = WaitBlock->Last;
if (NextWaitBlock != NULL) {
WaitBlock = NextWaitBlock;
break;
}
WaitBlock = WaitBlock->Next;
}
if (WaitBlock->ShareCount > 0) {
ASSERT (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
if (InterlockedDecrement (&WaitBlock->ShareCount) > 0) {
return;
}
}
}
while (1) {
if (OldValue.Waking) {
NewValue.Value = OldValue.Value & ~(EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_MULTIPLE_SHARED);
ASSERT (NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
return;
}
} else {
NewValue.Value = (OldValue.Value & ~(EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_MULTIPLE_SHARED)) |
EX_PUSH_LOCK_WAKING;
ASSERT (NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
TopValue = NewValue;
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
ExfWakePushLock (PushLock, TopValue);
return;
}
}
OldValue = NewValue;
}
}
}
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfAcquirePushLockShared (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Acquire a push lock shared
Arguments:
PushLock - Push lock to be acquired
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
EX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
BOOLEAN Optimize;
#if defined (USE_EXP_BACKOFF)
RTL_BACKOFF Backoff = {0};
#endif
OldValue = ReadForWriteAccess (PushLock);
while (1) {
//
// If the lock is already held we need to wait if its not held shared
//
if (!OldValue.Locked || (!OldValue.Waiting && OldValue.Shared > 0)) {
if (OldValue.Waiting) {
NewValue.Value = OldValue.Value + EX_PUSH_LOCK_LOCK;
} else {
NewValue.Value = (OldValue.Value + EX_PUSH_LOCK_SHARE_INC) | EX_PUSH_LOCK_LOCK;
}
ASSERT (NewValue.Locked);
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
break;
}
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
} else {
WaitBlock.Flags = EX_PUSH_LOCK_FLAGS_SPINNING;
WaitBlock.ShareCount = 0;
Optimize = FALSE;
WaitBlock.Previous = NULL;
//
// Move the sharecount to our wait block if need be.
//
if (OldValue.Waiting) {
WaitBlock.Last = NULL;
WaitBlock.Next = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & (EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_MULTIPLE_SHARED)) |
EX_PUSH_LOCK_WAITING | EX_PUSH_LOCK_WAKING);
if (!OldValue.Waking) {
Optimize = TRUE;
}
} else {
WaitBlock.Last = &WaitBlock;
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING);
}
ASSERT (NewValue.Waiting);
#if DBG
WaitBlock.Signaled = FALSE;
WaitBlock.OldValue = OldValue.Ptr;
WaitBlock.NewValue = NewValue.Ptr;
WaitBlock.PushLock = PushLock;
#endif
TopValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
ULONG i;
//
// If we set the waiting bit then optimize the list
//
if (Optimize) {
ExpOptimizePushLockList (PushLock, TopValue);
}
//
// It is safe to initialize the gate here, as the interlocked operation below forces
// a gate signal to always follow gate initialization.
//
KeInitializeGate (&WaitBlock.WakeGate);
for (i = ExPushLockSpinCount; i > 0; i--) {
if (((*(volatile LONG *)&WaitBlock.Flags)&EX_PUSH_LOCK_FLAGS_SPINNING) == 0) {
break;
}
KeYieldProcessor ();
}
if (InterlockedBitTestAndReset ((LONG*)&WaitBlock.Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeWaitForGate (&WaitBlock.WakeGate, WrPushLock, KernelMode);
#if DBG
ASSERT (WaitBlock.Signaled);
#endif
}
} else {
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
}
}
OldValue = NewValue;
}
}
