/*++
* @name ExfWakePushLock
*
* The ExfWakePushLock routine wakes a Pushlock that is in the waiting
* state.
*
* @param PushLock
* Pointer to a pushlock that is waiting.
*
* @param OldValue
* Last known value of the pushlock before this routine was called.
*
* @return None.
*
* @remarks This is an internal routine; do not call it manually. Only the system
* can properly know if the pushlock is ready to be awakened or not.
* External callers should use ExfTrytoWakePushLock.
*
*--*/
VOID
FASTCALL
ExfWakePushLock(PEX_PUSH_LOCK PushLock,
EX_PUSH_LOCK OldValue)
{
EX_PUSH_LOCK NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock, FirstWaitBlock, LastWaitBlock;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
KIRQL OldIrql;
/* Start main wake loop */
for (;;)
{
/* Sanity checks */
ASSERT(!OldValue.MultipleShared);
/* Check if it's locked */
while (OldValue.Locked)
{
/* It's not waking anymore */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */
ASSERT(!NewValue.Waking);
ASSERT(NewValue.Locked);
ASSERT(NewValue.Waiting);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
/* Someone changed the value behind our back, update it*/
OldValue = NewValue;
}
/* Save the First Block */
FirstWaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
WaitBlock = FirstWaitBlock;
/* Try to find the last block */
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
/* Check if we found it */
if (LastWaitBlock)
{
/* Use it */
WaitBlock = LastWaitBlock;
break;
}
/* Save the previous block */
PreviousWaitBlock = WaitBlock;
/* Move to next block */
WaitBlock = WaitBlock->Next;
/* Save the previous block */
WaitBlock->Previous = PreviousWaitBlock;
}
/* Check if the last Wait Block is not Exclusive or if it's the only one */
PreviousWaitBlock = WaitBlock->Previous;
if (!(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE) ||
!(PreviousWaitBlock))
{
/* Destroy the pushlock */
NewValue.Value = 0;
ASSERT(!NewValue.Waking);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) break;
/* Someone changed the value behind our back, update it*/
OldValue = NewValue;
}
else
{
/* Link the wait blocks */
FirstWaitBlock->Last = PreviousWaitBlock;
WaitBlock->Previous = NULL;
/* Sanity checks */
ASSERT(FirstWaitBlock != WaitBlock);
ASSERT(PushLock->Waiting);
/* Remove waking bit from pushlock */
InterlockedAndPointer(&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
/* Leave the loop */
break;
}
}
/* Check if there's a previous block */
OldIrql = DISPATCH_LEVEL;
if (WaitBlock->Previous)
{
/* Raise to Dispatch */
KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
}
/* Signaling loop */
for (;;)
{
/* Get the previous Wait block */
PreviousWaitBlock = WaitBlock->Previous;
/* Sanity check */
ASSERT(!WaitBlock->Signaled);
#if DBG
/* We are about to get signaled */
WaitBlock->Signaled = TRUE;
#endif
/* Set the Wait Bit in the Wait Block */
if (!InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Nobody signaled us, so do it */
KeSignalGateBoostPriority(&WaitBlock->WakeGate);
}
/* Set the wait block and check if there still is one to loop*/
WaitBlock = PreviousWaitBlock;
if (!WaitBlock) break;
}
/* Check if we have to lower back the IRQL */
if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);
}
NTKERNELAPI
VOID
FASTCALL
ExfWakePushLock (
IN PEX_PUSH_LOCK PushLock,
IN EX_PUSH_LOCK TopValue
)
/*++
Routine Description:
Walks the pushlock waiting list and wakes waiters if the lock is still unacquired.
Arguments:
PushLock - Push lock to be walked
TopValue - Start of the chain (*PushLock)
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock, FirstWaitBlock, PreviousWaitBlock;
KIRQL OldIrql;
OldValue = TopValue;
while (1) {
//
// Nobody should be walking the list while we manipulate it.
//
ASSERT (!OldValue.MultipleShared);
//
// No point waking somebody to find a locked lock. Just clear the waking bit
//
while (OldValue.Locked) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_WAKING;
ASSERT (!NewValue.Waking);
ASSERT (NewValue.Locked);
ASSERT (NewValue.Waiting);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
return;
}
OldValue = NewValue;
}
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~(ULONG_PTR)EX_PUSH_LOCK_PTR_BITS);
FirstWaitBlock = WaitBlock;
while (1) {
NextWaitBlock = WaitBlock->Last;
if (NextWaitBlock != NULL) {
WaitBlock = NextWaitBlock;
break;
}
PreviousWaitBlock = WaitBlock;
WaitBlock = WaitBlock->Next;
WaitBlock->Previous = PreviousWaitBlock;
}
if (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE &&
(PreviousWaitBlock = WaitBlock->Previous) != NULL) {
FirstWaitBlock->Last = PreviousWaitBlock;
WaitBlock->Previous = NULL;
ASSERT (FirstWaitBlock != WaitBlock);
ASSERT (PushLock->Waiting);
#if defined (_WIN64)
InterlockedAnd64 ((LONG64 *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
#else
InterlockedAnd ((LONG *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
#endif
break;
} else {
NewValue.Value = 0;
ASSERT (!NewValue.Waking);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
break;
}
OldValue = NewValue;
}
}
//
// If we are waking more than one thread then raise to DPC level to prevent us
// getting rescheduled part way through the operation
//
OldIrql = DISPATCH_LEVEL;
if (WaitBlock->Previous != NULL) {
KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
}
while (1) {
NextWaitBlock = WaitBlock->Previous;
#if DBG
ASSERT (!WaitBlock->Signaled);
WaitBlock->Signaled = TRUE;
#endif
if (!InterlockedBitTestAndReset (&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeSignalGateBoostPriority (&WaitBlock->WakeGate);
}
WaitBlock = NextWaitBlock;
if (WaitBlock == NULL) {
break;
}
}
if (OldIrql != DISPATCH_LEVEL) {
KeLowerIrql (OldIrql);
}
}
