bool VInterlocked::TestAndClear( sLONG *inValue, sLONG inBitNumber)
{
xbox_assert( inBitNumber >= 0 && inBitNumber <= 31);
#if VERSIONMAC
#if SMALLENDIAN
return ::OSAtomicTestAndClearBarrier( inBitNumber ^ 7, inValue);
#else
return ::OSAtomicTestAndClearBarrier( 31 - inBitNumber, inValue);
#endif
#elif VERSION_LINUX
uLONG mask=~(0x1<<inBitNumber);
sLONG val=__sync_fetch_and_and(inValue, mask);
return (val & mask ? true : false);
#elif VERSIONWIN
return InterlockedBitTestAndReset( (LONG*)inValue, inBitNumber) != 0;
#endif
}
VOID
AhciPortBusChangeDpcRoutine(
_In_ PSTOR_DPC Dpc,
_In_ PVOID AdapterExtension,
_In_opt_ PVOID SystemArgument1,
_In_opt_ PVOID SystemArgument2
)
/*++
Asks port driver to request a QDR on behalf of the miniport
It assumes:
nothing
Called by:
Indirectly by AhciHwInterrupt
It performs:
1 Kicks off the Start Channel state machine
2 Requests QDR
Affected Variables/Registers:
none
--*/
{
PAHCI_CHANNEL_EXTENSION channelExtension = (PAHCI_CHANNEL_EXTENSION)SystemArgument1;
ULONG busChangeInProcess;
BOOLEAN portIdle = FALSE;
UNREFERENCED_PARAMETER(Dpc);
UNREFERENCED_PARAMETER(AdapterExtension);
UNREFERENCED_PARAMETER(SystemArgument2);
if (channelExtension == NULL) {
NT_ASSERT(channelExtension != NULL);
return;
}
busChangeInProcess = InterlockedBitTestAndSet((LONG*)&channelExtension->PoFxPendingWork, 1); //BusChange is at bit 1
if (busChangeInProcess == 1) {
// bus change is pending in another process.
return;
}
RecordExecutionHistory(channelExtension, 0x00000030);//AhciPortBusChangeDpcRoutine
PortAcquireActiveReference(channelExtension, NULL, &portIdle);
// if port is in Active state, continue to process bus change notification.
// otherwise, it will be processed when port gets into Active state
if (!portIdle) {
ULONG busChangePending;
busChangePending = InterlockedBitTestAndReset((LONG*)&channelExtension->PoFxPendingWork, 1); //BusChange is at bit 1
if (busChangePending == 1) {
PortBusChangeProcess(channelExtension);
}
}
return;
}
/*++
* @name ExTimedWaitForUnblockPushLock
*
* The ExTimedWaitForUnblockPushLock routine waits for a pushlock
* to be unblocked, for a specified internal.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param WaitBlock
* Pointer to the pushlock's wait block.
*
* @param Timeout
* Amount of time to wait for this pushlock to be unblocked.
*
* @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
* code returned by KeWaitForSingleObject.
*
* @remarks If the wait fails, then a manual unblock is attempted.
*
*--*/
NTSTATUS
FASTCALL
ExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock,
IN PVOID WaitBlock,
IN PLARGE_INTEGER Timeout)
{
NTSTATUS Status;
/* Initialize the wait event */
KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->WakeEvent,
SynchronizationEvent,
FALSE);
#ifdef CONFIG_SMP
/* Spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
/* Check if we got lucky and can leave early */
if (!(*(volatile LONG *)&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags & EX_PUSH_LOCK_WAITING))
return STATUS_SUCCESS;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags,
EX_PUSH_LOCK_FLAGS_WAIT_V))
{
/* Nobody removed it already, let's do a full wait */
Status = KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->
WakeEvent,
WrPushLock,
KernelMode,
FALSE,
Timeout);
/* Check if the wait was satisfied */
if (Status != STATUS_SUCCESS)
{
/* Try unblocking the pushlock if it was not */
ExfUnblockPushLock(PushLock, WaitBlock);
}
}
else
{
/* Someone beat us to it, no need to wait */
Status = STATUS_SUCCESS;
}
/* Return status */
return Status;
}
/*++
* @name ExfUnblockPushLock
* @implemented NT5.1
*
* The ExfUnblockPushLock routine unblocks a previously blocked PushLock.
*
* @param PushLock
* Pointer to a previously blocked PushLock.
*
* @return None.
*
* @remarks Callers of ExfUnblockPushLock can be running at any IRQL.
*
*--*/
VOID
FASTCALL
ExfUnblockPushLock(PEX_PUSH_LOCK PushLock,
PVOID CurrentWaitBlock)
{
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock;
KIRQL OldIrql = DISPATCH_LEVEL;
/* Get the wait block and erase the previous one */
WaitBlock = InterlockedExchangePointer(&PushLock->Ptr, NULL);
if (WaitBlock)
{
/* Check if there is a linked pushlock and raise IRQL appropriately */
if (WaitBlock->Next) KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
/* Start block loop */
while (WaitBlock)
{
/* Get the next block */
NextWaitBlock = WaitBlock->Next;
/* Remove the wait flag from the Wait block */
if (!InterlockedBitTestAndReset(&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_WAIT_V))
{
/* Nobody removed the flag before us, so signal the event */
KeSetEventBoostPriority(&WaitBlock->WakeEvent, NULL);
}
/* Try the next one */
WaitBlock = NextWaitBlock;
}
/* Lower IRQL if needed */
if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);
}
/* Check if we got a wait block that's pending */
if ((CurrentWaitBlock) &&
(((PEX_PUSH_LOCK_WAIT_BLOCK)CurrentWaitBlock)->Flags &
EX_PUSH_LOCK_FLAGS_WAIT))
{
/* Wait for the pushlock to be unblocked */
ExWaitForUnblockPushLock(PushLock, CurrentWaitBlock);
}
}
int __cdecl main(int argc, char *argv[]) {
/*
* Initialize the PAL and return FAILURE if this fails
*/
if(0 != (PAL_Initialize(argc, argv)))
{
return FAIL;
}
for (int i = 0; i < sizeof (test_data) / sizeof (TEST_DATA); i++)
{
LONG baseVal = test_data[i].baseValue;
LONG bitPosition = test_data[i].bitPosition;
UCHAR ret = InterlockedBitTestAndReset(
&baseVal, /* Variable to manipulate */
bitPosition);
if (ret != test_data[i].expectedReturnValue)
{
Fail("ERROR: InterlockedBitTestAndReset(%d): Expected return value is %d,"
"Actual return value is %d.",
i,
test_data[i].expectedReturnValue,
ret);
}
if (baseVal != test_data[i].expectedValue)
{
Fail("ERROR: InterlockedBitTestAndReset(%d): Expected value is %x,"
"Actual value is %x.",
i,
test_data[i].expectedValue,
baseVal);
}
}
PAL_Terminate();
return PASS;
}
DWORD GSRasterizerMT::ThreadProc()
{
// _mm_setcsr(MXCSR);
while(!m_exit)
{
if(*m_sync & (1 << m_id))
{
__super::Draw(m_data);
InterlockedBitTestAndReset(m_sync, m_id);
}
else
{
_mm_pause();
}
}
return 0;
}
BOOL WINAPI
RtlpRemoveVectoredHandler(PVOID Handle,
ULONG Type)
{
LdrpVectorHandler* HandlerList = &LdrpVectorHandlerList[Type];
VEH_NODE *CurrentNode = (VEH_NODE*)HandlerList->Head.Flink;
fnRtlAcquireSRWLockExclusive(&HandlerList->Lock);
while (CurrentNode != (VEH_NODE*)&HandlerList->Head) {
if ((PVOID)CurrentNode != Handle) {
CurrentNode = (VEH_NODE*)CurrentNode->Entry.Flink;
continue;
}
// find the entry.
CurrentNode->RefCount--;
if (CurrentNode->RefCount == 0) {
PVOID Peb = NtCurrentPeb();
if (RemoveEntryList(&CurrentNode->Entry)) {
// list empty
InterlockedBitTestAndReset((LONG*)((PBYTE)Peb+0x28), // Peb.EnvironmentUpdateCount
Type + 2);
}
fnRtlReleaseSRWLockExclusive(&HandlerList->Head);
fnRtlFreeHeap(*(PVOID*)((PBYTE)Peb + 0x18),
0,
CurrentNode);
}
else {
fnRtlReleaseSRWLockExclusive(&HandlerList->Head);
}
return TRUE;
}
fnRtlReleaseSRWLockExclusive(&HandlerList->Head);
return FALSE;
}
NTKERNELAPI
NTSTATUS
FASTCALL
ExTimedWaitForUnblockPushLock (
__inout PEX_PUSH_LOCK PushLock,
__inout PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock,
__in_opt PLARGE_INTEGER Timeout
)
{
ULONG i;
NTSTATUS Status;
UNREFERENCED_PARAMETER (PushLock);
KeInitializeEvent (&WaitBlock->WakeEvent, SynchronizationEvent, FALSE);
for (i = ExPushLockSpinCount; i > 0; i--) {
if (((*(volatile LONG *)&WaitBlock->Flags)&EX_PUSH_LOCK_FLAGS_SPINNING) == 0) {
return STATUS_SUCCESS;
}
KeYieldProcessor ();
}
if (InterlockedBitTestAndReset (&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
Status = KeWaitForSingleObject (&WaitBlock->WakeEvent,
WrPushLock,
KernelMode,
FALSE,
Timeout);
if (Status != STATUS_SUCCESS) {
ExfUnblockPushLock (PushLock,
WaitBlock);
}
} else {
Status = STATUS_SUCCESS;
}
return Status;
}
NTSTATUS
NTAPI
x86BiosFreeBuffer(
USHORT Segment,
USHORT Offset)
{
/* Check if the system is initialized and if the address matches */
if (!x86BiosIsInitialized || Segment != 0x2000 || Offset != 0)
{
/* Something was wrong, fail */
return STATUS_INVALID_PARAMETER;
}
/* Check if the buffer was allocated */
if (!InterlockedBitTestAndReset(&x86BiosBufferIsAllocated, 0))
{
/* It was not, fail */
return STATUS_INVALID_PARAMETER;
}
/* Buffer is freed, nothing more to do */
return STATUS_SUCCESS;;
}
/*++
* @name ExfWakePushLock
*
* The ExfWakePushLock routine wakes a Pushlock that is in the waiting
* state.
*
* @param PushLock
* Pointer to a pushlock that is waiting.
*
* @param OldValue
* Last known value of the pushlock before this routine was called.
*
* @return None.
*
* @remarks This is an internal routine; do not call it manually. Only the system
* can properly know if the pushlock is ready to be awakened or not.
* External callers should use ExfTrytoWakePushLock.
*
*--*/
VOID
FASTCALL
ExfWakePushLock(PEX_PUSH_LOCK PushLock,
EX_PUSH_LOCK OldValue)
{
EX_PUSH_LOCK NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock, FirstWaitBlock, LastWaitBlock;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
KIRQL OldIrql;
/* Start main wake loop */
for (;;)
{
/* Sanity checks */
ASSERT(!OldValue.MultipleShared);
/* Check if it's locked */
while (OldValue.Locked)
{
/* It's not waking anymore */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */
ASSERT(!NewValue.Waking);
ASSERT(NewValue.Locked);
ASSERT(NewValue.Waiting);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
/* Someone changed the value behind our back, update it*/
OldValue = NewValue;
}
/* Save the First Block */
FirstWaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
WaitBlock = FirstWaitBlock;
/* Try to find the last block */
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
/* Check if we found it */
if (LastWaitBlock)
{
/* Use it */
WaitBlock = LastWaitBlock;
break;
}
/* Save the previous block */
PreviousWaitBlock = WaitBlock;
/* Move to next block */
WaitBlock = WaitBlock->Next;
/* Save the previous block */
WaitBlock->Previous = PreviousWaitBlock;
}
/* Check if the last Wait Block is not Exclusive or if it's the only one */
PreviousWaitBlock = WaitBlock->Previous;
if (!(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE) ||
!(PreviousWaitBlock))
{
/* Destroy the pushlock */
NewValue.Value = 0;
ASSERT(!NewValue.Waking);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) break;
/* Someone changed the value behind our back, update it*/
OldValue = NewValue;
}
else
{
/* Link the wait blocks */
FirstWaitBlock->Last = PreviousWaitBlock;
WaitBlock->Previous = NULL;
/* Sanity checks */
ASSERT(FirstWaitBlock != WaitBlock);
ASSERT(PushLock->Waiting);
/* Remove waking bit from pushlock */
InterlockedAndPointer(&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
/* Leave the loop */
break;
}
}
/* Check if there's a previous block */
OldIrql = DISPATCH_LEVEL;
if (WaitBlock->Previous)
{
/* Raise to Dispatch */
KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
}
/* Signaling loop */
for (;;)
{
/* Get the previous Wait block */
PreviousWaitBlock = WaitBlock->Previous;
/* Sanity check */
ASSERT(!WaitBlock->Signaled);
#if DBG
/* We are about to get signaled */
WaitBlock->Signaled = TRUE;
#endif
/* Set the Wait Bit in the Wait Block */
if (!InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Nobody signaled us, so do it */
KeSignalGateBoostPriority(&WaitBlock->WakeGate);
}
/* Set the wait block and check if there still is one to loop*/
WaitBlock = PreviousWaitBlock;
if (!WaitBlock) break;
}
/* Check if we have to lower back the IRQL */
if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);
}
/*++
* @name ExAcquirePushLockShared
* @implemented NT5.1
*
* The ExAcquirePushLockShared routine acquires a shared PushLock.
*
* @params PushLock
* Pointer to the pushlock which is to be acquired.
*
* @return None.
*
* @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeAcquireCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue;
BOOLEAN NeedWake;
EX_PUSH_LOCK_WAIT_BLOCK Block;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = &Block;
/* Start main loop */
for (;;)
{
/* Check if it's unlocked or if it's waiting without any sharers */
if (!(OldValue.Locked) || (!(OldValue.Waiting) && (OldValue.Shared > 0)))
{
/* Check if anyone is waiting on it */
if (!OldValue.Waiting)
{
/* Increase the share count and lock it */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
NewValue.Shared++;
}
else
{
/* Simply set the lock bit */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
}
/* Sanity check */
ASSERT(NewValue.Locked);
/* Set the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Break out of the loop */
break;
}
else
{
/* We'll have to create a Waitblock */
WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_WAIT;
WaitBlock->ShareCount = 0;
NeedWake = FALSE;
WaitBlock->Previous = NULL;
/* Check if there is already a waiter */
if (OldValue.Waiting)
{
/* Set the current Wait Block pointer */
WaitBlock->Next = (PEX_PUSH_LOCK_WAIT_BLOCK)(
OldValue.Value &~ EX_PUSH_LOCK_PTR_BITS);
/* Nobody is the last waiter yet */
WaitBlock->Last = NULL;
/* Point to ours */
NewValue.Value = (OldValue.Value & (EX_PUSH_LOCK_MULTIPLE_SHARED |
EX_PUSH_LOCK_LOCK)) |
EX_PUSH_LOCK_WAKING |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
/* Check if the pushlock was already waking */
if (!OldValue.Waking) NeedWake = TRUE;
}
else
{
/* We are the first waiter, so loop the wait block */
WaitBlock->Last = WaitBlock;
/* Point to our wait block */
NewValue.Value = (OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
}
/* Sanity check */
ASSERT(NewValue.Waiting);
#if DBG
/* Setup the Debug Wait Block */
WaitBlock->Signaled = 0;
WaitBlock->OldValue = OldValue;
WaitBlock->NewValue = NewValue;
WaitBlock->PushLock = PushLock;
#endif
/* Write the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr != OldValue.Ptr)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Update the value now */
OldValue = NewValue;
/* Check if the pushlock needed waking */
if (NeedWake)
{
/* Scan the Waiters and Wake PushLocks */
ExpOptimizePushLockList(PushLock, OldValue);
}
/* Set up the Wait Gate */
KeInitializeGate(&WaitBlock->WakeGate);
#ifdef CONFIG_SMP
/* Now spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
if (!(*(volatile LONG *)&WaitBlock->Flags & EX_PUSH_LOCK_WAITING))
break;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Fast-path did not work, we need to do a full wait */
KeWaitForGate(&WaitBlock->WakeGate, WrPushLock, KernelMode);
ASSERT(WaitBlock->Signaled);
}
/* We shouldn't be shared anymore */
ASSERT((WaitBlock->ShareCount == 0));
}
}
}
BOOL WINAPI
_RtlpCallVectoredHandler(PEXCEPTION_RECORD ExceptionRecord,
PCONTEXT ContextRecord,
ULONG Type)
{
PVOID Peb = NtCurrentPeb();
ULONG *pEnvironmentUpdateCount = (ULONG*)((PBYTE)Peb+0x28);
ULONG typeMask = 1 << (Type + 2);
BOOL Handled = FALSE;
LIST_ENTRY *needDeleteList = NULL;
EXCEPTION_POINTERS exceptionPointers;
LdrpVectorHandler *vectorHandler = &LdrpVectorHandlerList[Type];
if (((*pEnvironmentUpdateCount) & typeMask) == 0) {
return Handled;
}
needDeleteList = NULL;
exceptionPointers.ExceptionRecord = ExceptionRecord;
exceptionPointers.ContextRecord = ContextRecord;
fnRtlAcquireSRWLockExclusive(&vectorHandler->Lock);
VEH_NODE* currentNode = (VEH_NODE*)vectorHandler->Head.Flink;
while(¤tNode->Entry != &vectorHandler->Head) {
DWORD handlerResult;
VEH_NODE* oldNode = currentNode;
currentNode = (VEH_NODE*)currentNode->Entry.Flink;
oldNode->RefCount++;
fnRtlReleaseSRWLockExclusive(&vectorHandler->Lock);
PVECTORED_EXCEPTION_HANDLER handler = (PVECTORED_EXCEPTION_HANDLER)fnRtlDecodePointer(oldNode->Handler);
handlerResult = handler(&exceptionPointers);
fnRtlAcquireSRWLockExclusive(&vectorHandler->Lock);
oldNode->RefCount--;
if (oldNode->RefCount == 0) {
if (RemoveEntryList(&oldNode->Entry)) {
// list empty
InterlockedBitTestAndReset((volatile LONG *)pEnvironmentUpdateCount,
Type + 2);
}
oldNode->Entry.Flink = needDeleteList;
needDeleteList = &oldNode->Entry;
}
if (handlerResult == EXCEPTION_CONTINUE_EXECUTION) {
Handled = TRUE;
break;
}
}
fnRtlReleaseSRWLockExclusive(&vectorHandler->Lock);
while(needDeleteList) {
LIST_ENTRY* next = needDeleteList->Flink;
fnRtlFreeHeap(*(PVOID*)((PBYTE)Peb + 0x18),
0,
needDeleteList);
needDeleteList = next;
}
return Handled;
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfAcquirePushLockShared (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Acquire a push lock shared
Arguments:
PushLock - Push lock to be acquired
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
EX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
BOOLEAN Optimize;
#if defined (USE_EXP_BACKOFF)
RTL_BACKOFF Backoff = {0};
#endif
OldValue = ReadForWriteAccess (PushLock);
while (1) {
//
// If the lock is already held we need to wait if its not held shared
//
if (!OldValue.Locked || (!OldValue.Waiting && OldValue.Shared > 0)) {
if (OldValue.Waiting) {
NewValue.Value = OldValue.Value + EX_PUSH_LOCK_LOCK;
} else {
NewValue.Value = (OldValue.Value + EX_PUSH_LOCK_SHARE_INC) | EX_PUSH_LOCK_LOCK;
}
ASSERT (NewValue.Locked);
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
break;
}
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
} else {
WaitBlock.Flags = EX_PUSH_LOCK_FLAGS_SPINNING;
WaitBlock.ShareCount = 0;
Optimize = FALSE;
WaitBlock.Previous = NULL;
//
// Move the sharecount to our wait block if need be.
//
if (OldValue.Waiting) {
WaitBlock.Last = NULL;
WaitBlock.Next = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & (EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_MULTIPLE_SHARED)) |
EX_PUSH_LOCK_WAITING | EX_PUSH_LOCK_WAKING);
if (!OldValue.Waking) {
Optimize = TRUE;
}
} else {
WaitBlock.Last = &WaitBlock;
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING);
}
ASSERT (NewValue.Waiting);
#if DBG
WaitBlock.Signaled = FALSE;
WaitBlock.OldValue = OldValue.Ptr;
WaitBlock.NewValue = NewValue.Ptr;
WaitBlock.PushLock = PushLock;
#endif
TopValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
ULONG i;
//
// If we set the waiting bit then optimize the list
//
if (Optimize) {
ExpOptimizePushLockList (PushLock, TopValue);
}
//
// It is safe to initialize the gate here, as the interlocked operation below forces
// a gate signal to always follow gate initialization.
//
KeInitializeGate (&WaitBlock.WakeGate);
for (i = ExPushLockSpinCount; i > 0; i--) {
if (((*(volatile LONG *)&WaitBlock.Flags)&EX_PUSH_LOCK_FLAGS_SPINNING) == 0) {
break;
}
KeYieldProcessor ();
}
if (InterlockedBitTestAndReset ((LONG*)&WaitBlock.Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeWaitForGate (&WaitBlock.WakeGate, WrPushLock, KernelMode);
#if DBG
ASSERT (WaitBlock.Signaled);
#endif
}
} else {
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
}
}
OldValue = NewValue;
}
}
NTKERNELAPI
VOID
FASTCALL
ExfWakePushLock (
IN PEX_PUSH_LOCK PushLock,
IN EX_PUSH_LOCK TopValue
)
/*++
Routine Description:
Walks the pushlock waiting list and wakes waiters if the lock is still unacquired.
Arguments:
PushLock - Push lock to be walked
TopValue - Start of the chain (*PushLock)
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock, FirstWaitBlock, PreviousWaitBlock;
KIRQL OldIrql;
OldValue = TopValue;
while (1) {
//
// Nobody should be walking the list while we manipulate it.
//
ASSERT (!OldValue.MultipleShared);
//
// No point waking somebody to find a locked lock. Just clear the waking bit
//
while (OldValue.Locked) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_WAKING;
ASSERT (!NewValue.Waking);
ASSERT (NewValue.Locked);
ASSERT (NewValue.Waiting);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
return;
}
OldValue = NewValue;
}
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~(ULONG_PTR)EX_PUSH_LOCK_PTR_BITS);
FirstWaitBlock = WaitBlock;
while (1) {
NextWaitBlock = WaitBlock->Last;
if (NextWaitBlock != NULL) {
WaitBlock = NextWaitBlock;
break;
}
PreviousWaitBlock = WaitBlock;
WaitBlock = WaitBlock->Next;
WaitBlock->Previous = PreviousWaitBlock;
}
if (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE &&
(PreviousWaitBlock = WaitBlock->Previous) != NULL) {
FirstWaitBlock->Last = PreviousWaitBlock;
WaitBlock->Previous = NULL;
ASSERT (FirstWaitBlock != WaitBlock);
ASSERT (PushLock->Waiting);
#if defined (_WIN64)
InterlockedAnd64 ((LONG64 *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
#else
InterlockedAnd ((LONG *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
#endif
break;
} else {
NewValue.Value = 0;
ASSERT (!NewValue.Waking);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
break;
}
OldValue = NewValue;
}
}
//
// If we are waking more than one thread then raise to DPC level to prevent us
// getting rescheduled part way through the operation
//
OldIrql = DISPATCH_LEVEL;
if (WaitBlock->Previous != NULL) {
KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
}
while (1) {
NextWaitBlock = WaitBlock->Previous;
#if DBG
ASSERT (!WaitBlock->Signaled);
WaitBlock->Signaled = TRUE;
#endif
if (!InterlockedBitTestAndReset (&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeSignalGateBoostPriority (&WaitBlock->WakeGate);
}
WaitBlock = NextWaitBlock;
if (WaitBlock == NULL) {
break;
}
}
if (OldIrql != DISPATCH_LEVEL) {
KeLowerIrql (OldIrql);
}
}
NTKERNELAPI
VOID
FASTCALL
ExfUnblockPushLock (
__inout PEX_PUSH_LOCK PushLock,
__inout_opt PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock
)
/*++
Routine Description:
Unblock on a push lock
Arguments:
PushLock - Push lock to block on
WaitBlock - Wait block previously queued for waiting or NULL if there wasn't one
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue;
PEX_PUSH_LOCK_WAIT_BLOCK tWaitBlock;
KIRQL OldIrql;
//
// Pop the entire chain and wake them all up.
//
OldValue.Ptr = InterlockedExchangePointer (&PushLock->Ptr,
NULL);
tWaitBlock = OldValue.Ptr;
//
// If it looks like we will wake more than one thread then raise IRQL to prevent
// rescheduling during the operation.
//
OldIrql = DISPATCH_LEVEL;
if (tWaitBlock != NULL && tWaitBlock->Next != NULL) {
KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
}
while (tWaitBlock != NULL) {
OldValue.Ptr = tWaitBlock->Next;
if (!InterlockedBitTestAndReset (&tWaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeSetEventBoostPriority (&tWaitBlock->WakeEvent, NULL);
}
tWaitBlock = OldValue.Ptr;
}
if (OldIrql != DISPATCH_LEVEL) {
KeLowerIrql (OldIrql);
}
if (WaitBlock != NULL && (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_SPINNING) != 0) {
ExWaitForUnblockPushLock (PushLock, WaitBlock);
}
}
