static VALUE ir_compare_and_set(volatile VALUE self, VALUE expect_value, VALUE new_value) {
#if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050
if (OSAtomicCompareAndSwap64(expect_value, new_value, &DATA_PTR(self))) {
return Qtrue;
}
#elif defined(__sun)
/* Assuming VALUE is uintptr_t */
/* Based on the definition of uintptr_t from /usr/include/sys/int_types.h */
#if defined(_LP64) || defined(_I32LPx)
/* 64-bit: uintptr_t === unsigned long */
if (atomic_cas_ulong((uintptr_t *) &DATA_PTR(self), expect_value, new_value)) {
return Qtrue;
}
#else
/* 32-bit: uintptr_t === unsigned int */
if (atomic_cas_uint((uintptr_t *) &DATA_PTR(self), expect_value, new_value)) {
return Qtrue;
}
#endif
#elif defined _MSC_VER && defined _M_AMD64
if (InterlockedCompareExchange64((LONGLONG*)&DATA_PTR(self), new_value, expect_value)) {
return Qtrue;
}
#elif defined _MSC_VER && defined _M_IX86
if (InterlockedCompareExchange((LONG*)&DATA_PTR(self), new_value, expect_value)) {
return Qtrue;
}
#else
if (__sync_bool_compare_and_swap(&DATA_PTR(self), expect_value, new_value)) {
return Qtrue;
}
#endif
return Qfalse;
}
EFIAPI
InterlockedCompareExchangePointer (
IN OUT VOID **Value,
IN VOID *CompareValue,
IN VOID *ExchangeValue
)
{
UINT8 SizeOfValue;
SizeOfValue = sizeof (*Value);
switch (SizeOfValue) {
case sizeof (UINT32):
return (VOID*)(UINTN)InterlockedCompareExchange32 (
(UINT32*)Value,
(UINT32)(UINTN)CompareValue,
(UINT32)(UINTN)ExchangeValue
);
case sizeof (UINT64):
return (VOID*)(UINTN)InterlockedCompareExchange64 (
(UINT64*)Value,
(UINT64)(UINTN)CompareValue,
(UINT64)(UINTN)ExchangeValue
);
default:
ASSERT (FALSE);
return NULL;
}
}
__int64 _InterlockedOr64 (__int64 volatile *Destination,__int64 Value)
{
__int64 Old;
do {
Old = *Destination;
} while(InterlockedCompareExchange64(Destination,Old | Value,Old)!=Old);
return Old;
}
void baked_light_baker_add_64i(int64_t *dst,int64_t value) {
while(true) {
int64_t from = *dst;
int64_t to = from+value;
int64_t result = InterlockedCompareExchange64(dst,to,from);
if (result==from)
break;
}
}
INLINE void *compare_exchange_ptr(void * volatile *ptr, void *compare, void *exchange)
{
#ifdef PTR64
INT64 result = InterlockedCompareExchange64((LONGLONG *)ptr, (LONGLONG)exchange, (LONGLONG)compare);
return (void *)result;
#else
INT32 result = InterlockedCompareExchange((LPLONG)ptr, (LONG)exchange, (LONG)compare);
return (void *)result;
#endif
}
RU64
rInterlocked_get64
(
volatile RU64* pRu64
)
{
#ifdef RPAL_PLATFORM_WINDOWS
return InterlockedCompareExchange64( (LONG64*)pRu64, *pRu64, *pRu64 );
#elif defined( RPAL_PLATFORM_LINUX ) || defined( RPAL_PLATFORM_MACOSX )
return __sync_fetch_and_or( pRu64, 0 );
#endif
}
// static
void CycleTimer::InterlockedAddU64(unsigned __int64* loc, unsigned __int64 amount)
{
volatile __int64* vloc = (volatile __int64*)loc;
unsigned __int64 prev = *vloc;
for (;;)
{
unsigned __int64 next = prev + amount;
__int64 snext = (__int64)next;
__int64 sprev = (__int64)prev;
__int64 res = InterlockedCompareExchange64(vloc, snext, sprev);
if (res == sprev) return;
else prev = (unsigned __int64)res;
}
}
int64 SequenceNumber::GetNext()
{
int64 value = DateTime::Now().Ticks;
int64 oldValue;
do
{
oldValue = LastTimeStamp;
if (value <= oldValue)
{
value = oldValue + 1;
}
} while (InterlockedCompareExchange64(&LastTimeStamp, value, oldValue) != oldValue);
return value;
}
/* Wait for spin lock */
static int slwait (LOCKLONG *sl)
{
LOCKLONG Destination = *sl;
LONG Exchange = 1;
LONG Comparand = 0;
#if _WIN64
while (InterlockedCompareExchange64(&Destination, Exchange, Comparand))
#else
while (InterlockedCompareExchange (&Destination, Exchange, Comparand))
#endif
{
Sleep (0);
}
*sl = Destination;
return 0;
}
void baked_light_baker_add_64f(double *dst,double value) {
union {
int64_t i;
double f;
} swapy;
while(true) {
swapy.f=*dst;
int64_t from = swapy.i;
swapy.f+=value;
int64_t to=swapy.i;
int64_t result = InterlockedCompareExchange64((int64_t*)dst,to,from);
if (result==from)
break;
}
}
int64 ThreadInterlockedCompareExchange64( int64 volatile *pDest, int64 value, int64 comperand )
{
Assert( (size_t)pDest % 8 == 0 );
#if defined(_WIN64) || defined (_X360)
return InterlockedCompareExchange64( pDest, value, comperand );
#else
__asm
{
lea esi,comperand;
lea edi,value;
mov eax,[esi];
mov edx,4[esi];
mov ebx,[edi];
mov ecx,4[edi];
mov esi,pDest;
lock CMPXCHG8B [esi];
}
#endif
}
static inline bool compare_exchange(volatile Type& storage, Type& expected, Type desired) {
Type temp_expected = expected;
Type previous = static_cast<Type>(InterlockedCompareExchange64((__int64*)&storage, (__int64)desired, (__int64)temp_expected));
expected = previous;
return (previous == temp_expected);
}
bool compareAndExchange64(int64_t volatile* dest, int64_t exchange, int64_t comperand)
{
return InterlockedCompareExchange64(dest, exchange, comperand) == comperand;
}
//---------------------------------------------------------------------------
int64_t interlockedCompareExchange(volatile int64_t & v,int64_t exValue,int64_t cmpValue)
{
return InterlockedCompareExchange64((LONGLONG *) &v,exValue,cmpValue);
}
inline ::LONGLONG compare_exchange
( ::LONGLONG z, volatile ::LONGLONG& x, ::LONGLONG y )
{
return InterlockedCompareExchange64(&x, y, z);
}
int __cdecl main(int argc, char *argv[]) {
LONGLONG BaseVariableToManipulate = START_VALUE;
LONGLONG ValueToExchange = SECOND_VALUE;
LONGLONG TempValue;
LONGLONG TheReturn;
/*
* Initialize the PAL and return FAILURE if this fails
*/
if(0 != (PAL_Initialize(argc, argv)))
{
return FAIL;
}
/*
** Run only on 64 bit platforms
*/
#if defined(BIT64) && defined(PLATFORM_UNIX)
/* Compare START_VALUE with BaseVariableToManipulate, they're equal,
so exchange
*/
TheReturn = InterlockedCompareExchange64(
&BaseVariableToManipulate, /* Destination */
ValueToExchange, /* Exchange value */
START_VALUE); /* Compare value */
/* Exchanged, these should be equal now */
if(BaseVariableToManipulate != ValueToExchange)
{
#ifdef PLATFORM_UNIX
Fail("ERROR: A successful compare and exchange should have occurred, "
"making the variable have the value of %ll, as opposed to the "
"current value of %ll.",
ValueToExchange,BaseVariableToManipulate);
#else
Fail("ERROR: A successful compare and exchange should have occurred, "
"making the variable have the value of %I64, as opposed to the "
"current value of %d.",
ValueToExchange,BaseVariableToManipulate);
#endif
}
/* Check to make sure it returns the original number which
'BaseVariableToManipulate' was set to.
*/
if(TheReturn != START_VALUE)
{
#ifdef PLATFORM_UNIX
Fail("ERROR: The return value after the first exchange should be the "
"former value of the variable, which was %ll, but it is now %ll.",
START_VALUE,TheReturn);
#else
Fail("ERROR: The return value after the first exchange should be the "
"former value of the variable, which was %I64, but it is now %I64.",
START_VALUE,TheReturn);
#endif
}
ValueToExchange = THIRD_VALUE; /* Give this a new value */
TempValue = BaseVariableToManipulate; /* Note value of Base */
/*
Do an exchange where 'BaseVariableToManipulate' doesn't
match -- therefore the exchange shouldn't happen.
So, it should end up the same as the 'TempValue' we saved.
*/
InterlockedCompareExchange64(&BaseVariableToManipulate,
ValueToExchange,
START_VALUE);
if(BaseVariableToManipulate != TempValue)
{
#ifdef PLATFORM_UNIX
Fail("ERROR: An attempted exchange should have failed due to "
"the compare failing. But, it seems to have succeeded. The "
"value should be %ll but is %ll in this case.",
TempValue,BaseVariableToManipulate);
#else
Fail("ERROR: An attempted exchange should have failed due to "
"the compare failing. But, it seems to have succeeded. The "
"value should be %I64 but is %I64 in this case.",
TempValue,BaseVariableToManipulate);
#endif
}
#endif //if defined(BIT64) && defined(PLATFORM_UNIX)
PAL_Terminate();
return PASS;
}
BOOLEAN
ExRemoveHeadNBQueue (
IN PVOID Header,
OUT PULONG64 Value
)
/*++
Routine Description:
This function removes a queue entry from the head of the specified
non-blocking queue and returns the associated data value.
Arguments:
Header - Supplies an opaque pointer to a non-blocking queue header.
Value - Supplies a pointer to a variable that receives the queue
element value.
Return Value:
If an entry is removed from the specified non-blocking queue, then
TRUE is returned as the function value. Otherwise, FALSE is returned.
--*/
{
NBQUEUE_POINTER Head;
PNBQUEUE_NODE HeadNode;
NBQUEUE_POINTER Insert;
NBQUEUE_POINTER Next;
PNBQUEUE_NODE NextNode;
PNBQUEUE_HEADER QueueHead;
NBQUEUE_POINTER Tail;
PNBQUEUE_NODE TailNode;
//
// The following loop is executed until an entry can be removed from
// the specified non-blocking queue or until it can be determined that
// the queue is empty.
//
QueueHead = (PNBQUEUE_HEADER)Header;
do {
//
// Read the head queue pointer, the tail queue pointer, and the
// next queue pointer of the head queue pointer making sure the
// three pointers are coherent.
//
Head.Data = ReadForWriteAccess((volatile LONG64 *)(&QueueHead->Head.Data));
Tail.Data = *((volatile LONG64 *)(&QueueHead->Tail.Data));
HeadNode = UnpackNBQPointer(&Head);
Next.Data = *((volatile LONG64 *)(&HeadNode->Next.Data));
if (Head.Data == *((volatile LONG64 *)(&QueueHead->Head.Data))) {
//
// If the queue header node is equal to the queue tail node,
// then either the queue is empty or the tail pointer is falling
// behind. Otherwise, there is an entry in the queue that can
// be removed.
//
NextNode = UnpackNBQPointer(&Next);
TailNode = UnpackNBQPointer(&Tail);
if (HeadNode == TailNode) {
//
// If the next node of head pointer is NULL, then the queue
// is empty. Otherwise, attempt to move the tail forward.
//
if (NextNode == NULL) {
return FALSE;
} else {
PackNBQPointer(&Insert, NextNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
}
} else {
//
// There is an entry in the queue that can be removed.
//
*Value = NextNode->Value;
PackNBQPointer(&Insert, NextNode);
Insert.Count = Head.Count + 1;
if (InterlockedCompareExchange64(&QueueHead->Head.Data,
Insert.Data,
Head.Data) == Head.Data) {
break;
}
}
}
} while (TRUE);
//
// Free the node that was removed for the list by inserting the node
// in the associated SLIST.
//
InterlockedPushEntrySList(QueueHead->SlistHead,
(PSLIST_ENTRY)HeadNode);
return TRUE;
}
int TestInterlockedAccess(int argc, char* argv[])
{
int index;
LONG* Addend;
LONG* Target;
LONG oldValue;
LONG* Destination;
LONGLONG oldValue64;
LONGLONG* Destination64;
/* InterlockedIncrement */
Addend = _aligned_malloc(sizeof(LONG), sizeof(LONG));
if (!Addend)
{
printf("Failed to allocate memory\n");
return -1;
}
*Addend = 0;
for (index = 0; index < 10; index ++)
InterlockedIncrement(Addend);
if (*Addend != 10)
{
printf("InterlockedIncrement failure: Actual: %"PRId32", Expected: 10\n", *Addend);
return -1;
}
/* InterlockedDecrement */
for (index = 0; index < 10; index ++)
InterlockedDecrement(Addend);
if (*Addend != 0)
{
printf("InterlockedDecrement failure: Actual: %"PRId32", Expected: 0\n", *Addend);
return -1;
}
/* InterlockedExchange */
Target = _aligned_malloc(sizeof(LONG), sizeof(LONG));
if (!Target)
{
printf("Failed to allocate memory\n");
return -1;
}
*Target = 0xAA;
oldValue = InterlockedExchange(Target, 0xFF);
if (oldValue != 0xAA)
{
printf("InterlockedExchange failure: Actual: 0x%08"PRIX32", Expected: 0xAA\n", oldValue);
return -1;
}
if (*Target != 0xFF)
{
printf("InterlockedExchange failure: Actual: 0x%08"PRIX32", Expected: 0xFF\n", *Target);
return -1;
}
/* InterlockedExchangeAdd */
*Addend = 25;
oldValue = InterlockedExchangeAdd(Addend, 100);
if (oldValue != 25)
{
printf("InterlockedExchangeAdd failure: Actual: %"PRId32", Expected: 25\n", oldValue);
return -1;
}
if (*Addend != 125)
{
printf("InterlockedExchangeAdd failure: Actual: %"PRId32", Expected: 125\n", *Addend);
return -1;
}
/* InterlockedCompareExchange (*Destination == Comparand) */
Destination = _aligned_malloc(sizeof(LONG), sizeof(LONG));
if (!Destination)
{
printf("Failed to allocate memory\n");
return -1;
}
*Destination = 0xAABBCCDD;
oldValue = InterlockedCompareExchange(Destination, 0xCCDDEEFF, 0xAABBCCDD);
if (oldValue != 0xAABBCCDD)
{
printf("InterlockedCompareExchange failure: Actual: 0x%08"PRIX32", Expected: 0xAABBCCDD\n", oldValue);
return -1;
}
if (*Destination != 0xCCDDEEFF)
{
printf("InterlockedCompareExchange failure: Actual: 0x%08"PRIX32", Expected: 0xCCDDEEFF\n", *Destination);
return -1;
}
/* InterlockedCompareExchange (*Destination != Comparand) */
*Destination = 0xAABBCCDD;
oldValue = InterlockedCompareExchange(Destination, 0xCCDDEEFF, 0x66778899);
if (oldValue != 0xAABBCCDD)
{
printf("InterlockedCompareExchange failure: Actual: 0x%08"PRIX32", Expected: 0xAABBCCDD\n", oldValue);
return -1;
}
if (*Destination != 0xAABBCCDD)
{
printf("InterlockedCompareExchange failure: Actual: 0x%08"PRIX32", Expected: 0xAABBCCDD\n", *Destination);
return -1;
}
/* InterlockedCompareExchange64 (*Destination == Comparand) */
Destination64 = _aligned_malloc(sizeof(LONGLONG), sizeof(LONGLONG));
if (!Destination64)
{
printf("Failed to allocate memory\n");
return -1;
}
*Destination64 = 0x66778899AABBCCDD;
oldValue64 = InterlockedCompareExchange64(Destination64, 0x8899AABBCCDDEEFF, 0x66778899AABBCCDD);
if (oldValue64 != 0x66778899AABBCCDD)
{
printf("InterlockedCompareExchange failure: Actual: 0x%016"PRIX64", Expected: 0x66778899AABBCCDD\n", oldValue64);
return -1;
}
if (*Destination64 != 0x8899AABBCCDDEEFF)
{
printf("InterlockedCompareExchange failure: Actual: 0x%016"PRIX64", Expected: 0x8899AABBCCDDEEFF\n", *Destination64);
return -1;
}
/* InterlockedCompareExchange64 (*Destination != Comparand) */
*Destination64 = 0x66778899AABBCCDD;
oldValue64 = InterlockedCompareExchange64(Destination64, 0x8899AABBCCDDEEFF, 12345);
if (oldValue64 != 0x66778899AABBCCDD)
{
printf("InterlockedCompareExchange failure: Actual: 0x%016"PRIX64", Expected: 0x66778899AABBCCDD\n", oldValue64);
return -1;
}
if (*Destination64 != 0x66778899AABBCCDD)
{
printf("InterlockedCompareExchange failure: Actual: 0x%016"PRIX64", Expected: 0x66778899AABBCCDD\n", *Destination64);
return -1;
}
_aligned_free(Addend);
_aligned_free(Target);
_aligned_free(Destination);
_aligned_free(Destination64);
return 0;
}
BOOLEAN
ExInsertTailNBQueue (
IN PVOID Header,
IN ULONG64 Value
)
/*++
Routine Description:
This function inserts the specific data value at the tail of the
specified non-blocking queue.
Arguments:
Header - Supplies an opaque pointer to a non-blocking queue header.
Value - Supplies a pointer to an opaque data value.
Return Value:
If the specified opaque data value is successfully inserted at the tail
of the specified non-blocking queue, then a value of TRUE is returned as
the function value. Otherwise, a value of FALSE is returned.
N.B. FALSE is returned if a queue node cannot be allocated from the
associated SLIST.
--*/
{
NBQUEUE_POINTER Insert;
NBQUEUE_POINTER Next;
PNBQUEUE_NODE NextNode;
PNBQUEUE_HEADER QueueHead;
PNBQUEUE_NODE QueueNode;
NBQUEUE_POINTER Tail;
PNBQUEUE_NODE TailNode;
//
// Attempt to allocate a queue node from the SLIST associated with
// the specified non-blocking queue. If a node can be allocated, then
// the node is inserted at the tail of the specified non-blocking
// queue, and TRUE is returned as the function value. Otherwise, FALSE
// is returned.
//
QueueHead = (PNBQUEUE_HEADER)Header;
QueueNode = (PNBQUEUE_NODE)InterlockedPopEntrySList(QueueHead->SlistHead);
if (QueueNode != NULL) {
//
// Initialize the queue node next pointer and value.
//
QueueNode->Next.Data = 0;
QueueNode->Value = Value;
//
// The following loop is executed until the specified entry can
// be safely inserted at the tail of the specified non-blocking
// queue.
//
do {
//
// Read the tail queue pointer and the next queue pointer of
// the tail queue pointer making sure the two pointers are
// coherent.
//
Tail.Data = ReadForWriteAccess((volatile LONG64 *)(&QueueHead->Tail.Data));
TailNode = UnpackNBQPointer(&Tail);
Next.Data = *((volatile LONG64 *)(&TailNode->Next.Data));
QueueNode->Next.Count = Tail.Count + 1;
if (Tail.Data == *((volatile LONG64 *)(&QueueHead->Tail.Data))) {
//
// If the tail is pointing to the last node in the list,
// then attempt to insert the new node at the end of the
// list. Otherwise, the tail is not pointing to the last
// node in the list and an attempt is made to move the
// tail pointer to the next node.
//
NextNode = UnpackNBQPointer(&Next);
if (NextNode == NULL) {
PackNBQPointer(&Insert, QueueNode);
Insert.Count = Next.Count + 1;
if (InterlockedCompareExchange64(&TailNode->Next.Data,
Insert.Data,
Next.Data) == Next.Data) {
break;
}
} else {
PackNBQPointer(&Insert, NextNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
}
}
} while (TRUE);
//
// Attempt to move the tail to the new tail node.
//
PackNBQPointer(&Insert, QueueNode);
Insert.Count = Tail.Count + 1;
InterlockedCompareExchange64(&QueueHead->Tail.Data,
Insert.Data,
Tail.Data);
return TRUE;
} else {
return FALSE;
}
}
bool TRI_CompareIntegerUInt64 (volatile uint64_t* theValue, uint64_t oldValue) {
return ( (uint64_t)(InterlockedCompareExchange64((volatile LONGLONG*)(theValue), (LONGLONG)(oldValue), (LONGLONG)(oldValue) ) ) == oldValue );
}
bool TRI_CompareAndSwapIntegerInt64 (volatile int64_t* theValue, int64_t oldValue, int64_t newValue) {
return ( (int64_t)(InterlockedCompareExchange64((volatile LONGLONG*)(theValue), (LONGLONG)(newValue), (LONGLONG)(oldValue) ) ) == oldValue );
}
INT64 win_compare_exchange64(INT64 volatile *ptr, INT64 compare, INT64 exchange)
{
return InterlockedCompareExchange64((LONGLONG*)ptr, (LONGLONG)exchange, (LONGLONG)compare);
}
intptr_t CompareAndSwap(volatile intptr_t *ptr, intptr_t old_value, intptr_t new_value)
{
return InterlockedCompareExchange64((LONGLONG *)ptr, new_value, old_value);
}
