/*++
* @name ExfTryToWakePushLock
* @implemented NT5.2
*
* The ExfTryToWakePushLock attemps to wake a waiting pushlock.
*
* @param PushLock
* Pointer to a PushLock which is in the wait state.
*
* @return None.
*
* @remarks The pushlock must be in a wait state and must not be already waking.
*
*--*/
VOID
FASTCALL
ExfTryToWakePushLock(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue;
/*
* If the Pushlock is not waiting on anything, or if it's already waking up
* and locked, don't do anything
*/
if ((OldValue.Waking) || (OldValue.Locked) || !(OldValue.Waiting)) return;
/* Make it Waking */
NewValue = OldValue;
NewValue.Waking = TRUE;
/* Write the New Value */
if (InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr) == OldValue.Ptr)
{
/* Wake the Pushlock */
ExfWakePushLock(PushLock, NewValue);
}
}
/*++
* ExfReleasePushLockExclusive
* @implemented NT5.2
*
* The ExfReleasePushLockExclusive routine releases a previously
* exclusively acquired PushLock.
*
* @params PushLock
* Pointer to a previously acquired pushlock.
*
* @return None.
*
* @remarks Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeLeaveCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK NewValue, WakeValue;
EX_PUSH_LOCK OldValue = *PushLock;
/* Loop until we can change */
for (;;)
{
/* Sanity checks */
ASSERT(OldValue.Locked);
ASSERT(OldValue.Waiting || OldValue.Shared == 0);
/* Check if it's waiting and not yet waking */
if ((OldValue.Waiting) && !(OldValue.Waking))
{
/* Remove the lock bit, and add the wake bit */
NewValue.Value = (OldValue.Value &~ EX_PUSH_LOCK_LOCK) |
EX_PUSH_LOCK_WAKING;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked);
/* Write the New Value. Save our original value for waking */
WakeValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
/* Check if the value changed behind our back */
if (NewValue.Value == OldValue.Value)
{
/* Wake the Pushlock */
ExfWakePushLock(PushLock, WakeValue);
break;
}
}
else
{
/* A simple unlock */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_LOCK;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Waiting);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
/* Check if the value changed behind our back */
if (NewValue.Value == OldValue.Value) break;
}
/* Loop again */
OldValue = NewValue;
}
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfReleasePushLockExclusive (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Release a push lock that was acquired exclusive
Arguments:
PushLock - Push lock to be released
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
OldValue = ReadForWriteAccess (PushLock);
while (1) {
ASSERT (OldValue.Locked);
ASSERT (OldValue.Waiting || OldValue.Shared == 0);
if (OldValue.Waiting && !OldValue.Waking) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_LOCK + EX_PUSH_LOCK_WAKING;
ASSERT (NewValue.Waking && !NewValue.Locked);
TopValue = NewValue;
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
ExfWakePushLock (PushLock, TopValue);
break;
}
} else {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_LOCK;
ASSERT (NewValue.Waking || !NewValue.Waiting);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
break;
}
}
OldValue = NewValue;
}
}
VOID
FASTCALL
ExfTryToWakePushLock (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Tries to set the wake bit and wake the pushlock
Arguments:
PushLock - Push lock to be woken
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
OldValue = *PushLock;
if (OldValue.Waking || OldValue.Locked || !OldValue.Waiting) {
return;
}
NewValue.Value = OldValue.Value + EX_PUSH_LOCK_WAKING;
if (InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr) == OldValue.Ptr) {
ExfWakePushLock (PushLock, NewValue);
}
}
/*++
* @name ExfReleasePushLockShared
* @implemented NT5.2
*
* The ExfReleasePushLockShared macro releases a previously acquired PushLock.
*
* @params PushLock
* Pointer to a previously acquired pushlock.
*
* @return None.
*
* @remarks Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeLeaveCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfReleasePushLockShared(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue, WakeValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock;
/* Check if someone is waiting on the lock */
while (!OldValue.Waiting)
{
/* Check if it's shared */
if (OldValue.Shared > 1)
{
/* Write the Old Value but decrease share count */
NewValue = OldValue;
NewValue.Shared--;
}
else
{
/* Simply clear the lock */
NewValue.Value = 0;
}
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
/* Did it enter a wait state? */
OldValue = NewValue;
}
/* Ok, we do know someone is waiting on it. Are there more then one? */
if (OldValue.MultipleShared)
{
/* Get the wait block */
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
/* Loop until we find the last wait block */
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
/* Did it exist? */
if (LastWaitBlock)
{
/* Choose it */
WaitBlock = LastWaitBlock;
break;
}
/* Keep searching */
WaitBlock = WaitBlock->Next;
}
/* Sanity checks */
ASSERT(WaitBlock->ShareCount > 0);
ASSERT(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
/* Do the decrease and check if the lock isn't shared anymore */
if (InterlockedDecrement(&WaitBlock->ShareCount) > 0) return;
}
/*
* If nobody was waiting on the block, then we possibly reduced the number
* of times the pushlock was shared, and we unlocked it.
* If someone was waiting, and more then one person is waiting, then we
* reduced the number of times the pushlock is shared in the wait block.
* Therefore, at this point, we can now 'satisfy' the wait.
*/
for (;;)
{
/* Now we need to see if it's waking */
if (OldValue.Waking)
{
/* Remove the lock and multiple shared bits */
NewValue.Value = OldValue.Value;
NewValue.MultipleShared = FALSE;
NewValue.Locked = FALSE;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
}
else
{
/* Remove the lock and multiple shared bits */
NewValue.Value = OldValue.Value;
NewValue.MultipleShared = FALSE;
NewValue.Locked = FALSE;
/* It's not already waking, so add the wake bit */
NewValue.Waking = TRUE;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */
WakeValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value) continue;
/* The write was successful. The pushlock is Unlocked and Waking */
ExfWakePushLock(PushLock, WakeValue);
return;
}
}
}
/*++
* @name ExpOptimizePushLockList
*
* The ExpOptimizePushLockList routine optimizes the list of waiters
* associated to a pushlock's wait block.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param OldValue
* Last known value of the pushlock before this routine was called.
*
* @return None.
*
* @remarks At the end of the optimization, the pushlock will also be wakened.
*
*--*/
VOID
FASTCALL
ExpOptimizePushLockList(PEX_PUSH_LOCK PushLock,
EX_PUSH_LOCK OldValue)
{
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock, PreviousWaitBlock, FirstWaitBlock;
EX_PUSH_LOCK NewValue;
/* Start main loop */
for (;;)
{
/* Check if we've been unlocked */
if (!OldValue.Locked)
{
/* Wake us up and leave */
ExfWakePushLock(PushLock, OldValue);
break;
}
/* Get the wait block */
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
/* Loop the blocks */
FirstWaitBlock = WaitBlock;
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
if (LastWaitBlock)
{
/* Set this as the new last block, we're done */
FirstWaitBlock->Last = LastWaitBlock;
break;
}
/* Save the block */
PreviousWaitBlock = WaitBlock;
/* Get the next block */
WaitBlock = WaitBlock->Next;
/* Save the previous */
WaitBlock->Previous = PreviousWaitBlock;
}
/* Remove the wake bit */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */
ASSERT(NewValue.Locked);
ASSERT(!NewValue.Waking);
/* Update the value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
/* If we updated correctly, leave */
if (NewValue.Value == OldValue.Value) break;
/* Update value */
OldValue = NewValue;
}
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfReleasePushLock (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Release a push lock that was acquired exclusively or shared
Arguments:
PushLock - Push lock to be released
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock;
OldValue = ReadForWriteAccess (PushLock);
ASSERT (OldValue.Locked);
while (1) {
if (!OldValue.Waiting) {
if (OldValue.Shared > 1) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_SHARE_INC;
} else {
NewValue.Value = 0;
}
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
return;
}
OldValue = NewValue;
} else {
//
// See if we need to walk the list to find the share count
//
if (OldValue.MultipleShared) {
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK) (OldValue.Value & ~(ULONG_PTR)EX_PUSH_LOCK_PTR_BITS);
while (1) {
NextWaitBlock = WaitBlock->Last;
if (NextWaitBlock != NULL) {
WaitBlock = NextWaitBlock;
break;
}
WaitBlock = WaitBlock->Next;
}
if (WaitBlock->ShareCount > 0) {
ASSERT (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
if (InterlockedDecrement (&WaitBlock->ShareCount) > 0) {
return;
}
}
}
while (1) {
if (OldValue.Waking) {
NewValue.Value = OldValue.Value & ~(EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_MULTIPLE_SHARED);
ASSERT (NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
return;
}
} else {
NewValue.Value = (OldValue.Value & ~(EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_MULTIPLE_SHARED)) |
EX_PUSH_LOCK_WAKING;
ASSERT (NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
TopValue = NewValue;
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
ExfWakePushLock (PushLock, TopValue);
return;
}
}
OldValue = NewValue;
}
}
}
}
VOID
FASTCALL
ExpOptimizePushLockList (
IN PEX_PUSH_LOCK PushLock,
IN EX_PUSH_LOCK TopValue
)
/*++
Routine Description:
Walks the pushlock waiting list during contention and optimizes it
Arguments:
PushLock - Push lock to be walked
TopValue - Start of the chain (*PushLock)
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, PreviousWaitBlock, FirstWaitBlock, NextWaitBlock;
OldValue = TopValue;
while (1) {
if (!OldValue.Locked) {
ExfWakePushLock (PushLock, OldValue);
break;
}
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value & ~(EX_PUSH_LOCK_PTR_BITS));
FirstWaitBlock = WaitBlock;
while (1) {
NextWaitBlock = WaitBlock->Last;
if (NextWaitBlock != NULL) {
FirstWaitBlock->Last = NextWaitBlock;
break;
}
PreviousWaitBlock = WaitBlock;
WaitBlock = WaitBlock->Next;
WaitBlock->Previous = PreviousWaitBlock;
}
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_WAKING;
ASSERT (NewValue.Locked);
ASSERT (!NewValue.Waking);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
break;
}
OldValue = NewValue;
}
}
