/*++
* @name ExAcquirePushLockShared
* @implemented NT5.1
*
* The ExAcquirePushLockShared routine acquires a shared PushLock.
*
* @params PushLock
* Pointer to the pushlock which is to be acquired.
*
* @return None.
*
* @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeAcquireCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue;
BOOLEAN NeedWake;
EX_PUSH_LOCK_WAIT_BLOCK Block;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = &Block;
/* Start main loop */
for (;;)
{
/* Check if it's unlocked or if it's waiting without any sharers */
if (!(OldValue.Locked) || (!(OldValue.Waiting) && (OldValue.Shared > 0)))
{
/* Check if anyone is waiting on it */
if (!OldValue.Waiting)
{
/* Increase the share count and lock it */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
NewValue.Shared++;
}
else
{
/* Simply set the lock bit */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
}
/* Sanity check */
ASSERT(NewValue.Locked);
/* Set the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Break out of the loop */
break;
}
else
{
/* We'll have to create a Waitblock */
WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_WAIT;
WaitBlock->ShareCount = 0;
NeedWake = FALSE;
WaitBlock->Previous = NULL;
/* Check if there is already a waiter */
if (OldValue.Waiting)
{
/* Set the current Wait Block pointer */
WaitBlock->Next = (PEX_PUSH_LOCK_WAIT_BLOCK)(
OldValue.Value &~ EX_PUSH_LOCK_PTR_BITS);
/* Nobody is the last waiter yet */
WaitBlock->Last = NULL;
/* Point to ours */
NewValue.Value = (OldValue.Value & (EX_PUSH_LOCK_MULTIPLE_SHARED |
EX_PUSH_LOCK_LOCK)) |
EX_PUSH_LOCK_WAKING |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
/* Check if the pushlock was already waking */
if (!OldValue.Waking) NeedWake = TRUE;
}
else
{
/* We are the first waiter, so loop the wait block */
WaitBlock->Last = WaitBlock;
/* Point to our wait block */
NewValue.Value = (OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
}
/* Sanity check */
ASSERT(NewValue.Waiting);
#if DBG
/* Setup the Debug Wait Block */
WaitBlock->Signaled = 0;
WaitBlock->OldValue = OldValue;
WaitBlock->NewValue = NewValue;
WaitBlock->PushLock = PushLock;
#endif
/* Write the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr != OldValue.Ptr)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Update the value now */
OldValue = NewValue;
/* Check if the pushlock needed waking */
if (NeedWake)
{
/* Scan the Waiters and Wake PushLocks */
ExpOptimizePushLockList(PushLock, OldValue);
}
/* Set up the Wait Gate */
KeInitializeGate(&WaitBlock->WakeGate);
#ifdef CONFIG_SMP
/* Now spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
if (!(*(volatile LONG *)&WaitBlock->Flags & EX_PUSH_LOCK_WAITING))
break;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Fast-path did not work, we need to do a full wait */
KeWaitForGate(&WaitBlock->WakeGate, WrPushLock, KernelMode);
ASSERT(WaitBlock->Signaled);
}
/* We shouldn't be shared anymore */
ASSERT((WaitBlock->ShareCount == 0));
}
}
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfAcquirePushLockShared (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Acquire a push lock shared
Arguments:
PushLock - Push lock to be acquired
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
EX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
BOOLEAN Optimize;
#if defined (USE_EXP_BACKOFF)
RTL_BACKOFF Backoff = {0};
#endif
OldValue = ReadForWriteAccess (PushLock);
while (1) {
//
// If the lock is already held we need to wait if its not held shared
//
if (!OldValue.Locked || (!OldValue.Waiting && OldValue.Shared > 0)) {
if (OldValue.Waiting) {
NewValue.Value = OldValue.Value + EX_PUSH_LOCK_LOCK;
} else {
NewValue.Value = (OldValue.Value + EX_PUSH_LOCK_SHARE_INC) | EX_PUSH_LOCK_LOCK;
}
ASSERT (NewValue.Locked);
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
break;
}
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
} else {
WaitBlock.Flags = EX_PUSH_LOCK_FLAGS_SPINNING;
WaitBlock.ShareCount = 0;
Optimize = FALSE;
WaitBlock.Previous = NULL;
//
// Move the sharecount to our wait block if need be.
//
if (OldValue.Waiting) {
WaitBlock.Last = NULL;
WaitBlock.Next = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & (EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_MULTIPLE_SHARED)) |
EX_PUSH_LOCK_WAITING | EX_PUSH_LOCK_WAKING);
if (!OldValue.Waking) {
Optimize = TRUE;
}
} else {
WaitBlock.Last = &WaitBlock;
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING);
}
ASSERT (NewValue.Waiting);
#if DBG
WaitBlock.Signaled = FALSE;
WaitBlock.OldValue = OldValue.Ptr;
WaitBlock.NewValue = NewValue.Ptr;
WaitBlock.PushLock = PushLock;
#endif
TopValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
ULONG i;
//
// If we set the waiting bit then optimize the list
//
if (Optimize) {
ExpOptimizePushLockList (PushLock, TopValue);
}
//
// It is safe to initialize the gate here, as the interlocked operation below forces
// a gate signal to always follow gate initialization.
//
KeInitializeGate (&WaitBlock.WakeGate);
for (i = ExPushLockSpinCount; i > 0; i--) {
if (((*(volatile LONG *)&WaitBlock.Flags)&EX_PUSH_LOCK_FLAGS_SPINNING) == 0) {
break;
}
KeYieldProcessor ();
}
if (InterlockedBitTestAndReset ((LONG*)&WaitBlock.Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeWaitForGate (&WaitBlock.WakeGate, WrPushLock, KernelMode);
#if DBG
ASSERT (WaitBlock.Signaled);
#endif
}
} else {
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
}
}
OldValue = NewValue;
}
}