// "Increment" the value of a semaphore atomically and
// return its old value. Note that this implements
// the special case of "incrementing" any negative
// value to +1 directly.
//
// NOTE: The value will _not_ wrap above SEM_VALUE_MAX
static int __sem_inc(atomic_uint* sem_count_ptr) {
unsigned int old_value = atomic_load_explicit(sem_count_ptr, memory_order_relaxed);
unsigned int shared = old_value & SEMCOUNT_SHARED_MASK;
unsigned int new_value;
// Use memory_order_seq_cst in atomic_compare_exchange operation to ensure all
// memory access made before can be seen in other threads.
// A release fence may be sufficient, but it is still in discussion whether
// POSIX semaphores should provide sequential consistency.
do {
// Can't go higher than SEM_VALUE_MAX.
if (SEMCOUNT_TO_VALUE(old_value) == SEM_VALUE_MAX) {
break;
}
// If the counter is negative, go directly to one, otherwise just increment.
if (SEMCOUNT_TO_VALUE(old_value) < 0) {
new_value = SEMCOUNT_ONE | shared;
} else {
new_value = SEMCOUNT_INCREMENT(old_value) | shared;
}
} while (!atomic_compare_exchange_weak(sem_count_ptr, &old_value,
new_value));
return SEMCOUNT_TO_VALUE(old_value);
}
void *f(void* thr_data)
{
int my_idx = *(int*)thr_data;
cpu_set_t set;
CPU_ZERO(&set);
CPU_SET(my_idx, &set);
if( pthread_setaffinity_np(pthread_self(), sizeof(set), &set) ){
abort();
}
while( !start );
timings[my_idx][0] = GET_TS();
for(int n = 0; n < niter; ++n) {
_Bool flag = false;
while( !flag ){
int tmp = acnt;
flag = atomic_compare_exchange_weak(&acnt, &tmp, tmp + 1);
}
++cnt; // undefined behavior, in practice some updates missed
}
timings[my_idx][1] = GET_TS();
return 0;
}
TEST(stdatomic, atomic_compare_exchange) {
atomic_int i;
int expected;
atomic_store(&i, 123);
expected = 123;
ASSERT_TRUE(atomic_compare_exchange_strong(&i, &expected, 456));
ASSERT_FALSE(atomic_compare_exchange_strong(&i, &expected, 456));
ASSERT_EQ(456, expected);
atomic_store(&i, 123);
expected = 123;
ASSERT_TRUE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_FALSE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_EQ(456, expected);
atomic_store(&i, 123);
expected = 123;
int iter_count = 0;
do {
++iter_count;
ASSERT_LT(iter_count, 100); // Arbitrary limit on spurious compare_exchange failures.
ASSERT_EQ(expected, 123);
} while(!atomic_compare_exchange_weak(&i, &expected, 456));
ASSERT_FALSE(atomic_compare_exchange_weak(&i, &expected, 456));
ASSERT_EQ(456, expected);
atomic_store(&i, 123);
expected = 123;
iter_count = 0;
do {
++iter_count;
ASSERT_LT(iter_count, 100);
ASSERT_EQ(expected, 123);
} while(!atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_FALSE(atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_EQ(456, expected);
}
void CRYPTO_refcount_inc(CRYPTO_refcount_t *in_count) {
_Atomic CRYPTO_refcount_t *count = (_Atomic CRYPTO_refcount_t *) in_count;
uint32_t expected = atomic_load(count);
while (expected != CRYPTO_REFCOUNT_MAX) {
uint32_t new_value = expected + 1;
if (atomic_compare_exchange_weak(count, &expected, new_value)) {
break;
}
}
}
int main() {
std::shared_ptr < int > a;
std::shared_ptr < int > b;
std::shared_ptr < int > c = std::make_shared < int > (10);
while(atomic_compare_exchange_weak(&a, &b, c))
;
assert(atomic_load(&a) == c);
assert(atomic_load(&a).use_count() == 2);
}
int pthread_key_create(pthread_key_t* key, void (*key_destructor)(void*)) {
for (size_t i = 0; i < BIONIC_PTHREAD_KEY_COUNT; ++i) {
uintptr_t seq = atomic_load_explicit(&key_map[i].seq, memory_order_relaxed);
while (!SeqOfKeyInUse(seq)) {
if (atomic_compare_exchange_weak(&key_map[i].seq, &seq, seq + SEQ_INCREMENT_STEP)) {
atomic_store(&key_map[i].key_destructor, reinterpret_cast<uintptr_t>(key_destructor));
*key = i | KEY_VALID_FLAG;
return 0;
}
}
}
return EAGAIN;
}
static unsigned int
AcquireSegmentBufferSpace(
_In_ SystemPipe_t* Pipe,
_In_ SystemPipeSegmentBuffer_t* Buffer,
_In_ size_t Length)
{
// Variables
unsigned int ReadIndex;
unsigned int WriteIndex;
size_t BytesAvailable;
// Make sure we write all the bytes
while (1) {
WriteIndex = atomic_load(&Buffer->WritePointer);
ReadIndex = atomic_load(&Buffer->ReadCommitted);
BytesAvailable = MIN(
CalculateBytesAvailableForWriting(Buffer, ReadIndex, WriteIndex), Length);
if (BytesAvailable != Length) {
SchedulerAtomicThreadSleep((atomic_int*)&Buffer->ReadCommitted, (int*)&ReadIndex, 0);
continue; // Start over
}
// Synchronize with other producers
if (Pipe->Configuration & PIPE_MULTIPLE_PRODUCERS) {
while (BytesAvailable == Length) {
size_t NewWritePointer = WriteIndex + BytesAvailable;
if (atomic_compare_exchange_weak(&Buffer->WritePointer, &WriteIndex, NewWritePointer)) {
break;
}
ReadIndex = atomic_load(&Buffer->ReadCommitted);
BytesAvailable = MIN(
CalculateBytesAvailableForWriting(Buffer, ReadIndex, WriteIndex), Length);
}
// Did we end up overcomitting?
if (BytesAvailable != Length) {
continue; // Start write loop all over
}
}
else {
atomic_store_explicit(&Buffer->WritePointer, WriteIndex + BytesAvailable, memory_order_relaxed);
}
// Break us out here
if (BytesAvailable == Length) {
break;
}
}
return WriteIndex;
}
// Same as __sem_dec, but will not touch anything if the
// value is already negative *or* 0. Returns the old value.
static int __sem_trydec(atomic_uint* sem_count_ptr) {
unsigned int old_value = atomic_load_explicit(sem_count_ptr, memory_order_relaxed);
unsigned int shared = old_value & SEMCOUNT_SHARED_MASK;
// Use memory_order_seq_cst in atomic_compare_exchange operation to ensure all
// memory access made by other threads can be seen in current thread.
// An acquire fence may be sufficient, but it is still in discussion whether
// POSIX semaphores should provide sequential consistency.
do {
if (SEMCOUNT_TO_VALUE(old_value) <= 0) {
break;
}
} while (!atomic_compare_exchange_weak(sem_count_ptr, &old_value,
SEMCOUNT_DECREMENT(old_value) | shared));
return SEMCOUNT_TO_VALUE(old_value);
}
int CRYPTO_refcount_dec_and_test_zero(CRYPTO_refcount_t *in_count) {
_Atomic CRYPTO_refcount_t *count = (_Atomic CRYPTO_refcount_t *)in_count;
uint32_t expected = atomic_load(count);
for (;;) {
if (expected == 0) {
abort();
} else if (expected == CRYPTO_REFCOUNT_MAX) {
return 0;
} else {
const uint32_t new_value = expected - 1;
if (atomic_compare_exchange_weak(count, &expected, new_value)) {
return new_value == 0;
}
}
}
}
/*
* qsbr_register: register the current thread for QSBR.
*/
int
qsbr_register(qsbr_t *qs)
{
qsbr_tls_t *t, *head;
t = pthread_getspecific(qs->tls_key);
if (__predict_false(t == NULL)) {
if ((t = malloc(sizeof(qsbr_tls_t))) == NULL) {
return ENOMEM;
}
pthread_setspecific(qs->tls_key, t);
}
memset(t, 0, sizeof(qsbr_tls_t));
do {
head = qs->list;
t->next = head;
} while (!atomic_compare_exchange_weak(&qs->list, head, t));
return 0;
}
/////////////////////////////////////////////////////////////////////////
// System Pipe Structured Buffer Code
static void
GetSegmentProductionSpot(
_In_ SystemPipe_t* Pipe,
_In_ SystemPipeSegment_t* Segment)
{
// Variables
int ProductionSpots;
atomic_fetch_add(&Segment->References, 1);
while (1) {
ProductionSpots = atomic_load(&Segment->ProductionSpots);
if (!ProductionSpots) {
SchedulerAtomicThreadSleep(&Segment->ProductionSpots, &ProductionSpots, 0);
continue; // Start over
}
// Synchronize with other producers
if (Pipe->Configuration & PIPE_MULTIPLE_PRODUCERS) {
while (ProductionSpots) {
if (atomic_compare_exchange_weak(&Segment->ProductionSpots,
&ProductionSpots, ProductionSpots - 1)) {
break;
}
}
// Did we end up overcomitting?
if (!ProductionSpots) {
continue; // Start write loop all over
}
break;
}
else {
// No sweat
atomic_store_explicit(&Segment->ProductionSpots, ProductionSpots - 1, memory_order_relaxed);
break;
}
}
}
static size_t
ReadRawSegmentBuffer(
_In_ SystemPipe_t* Pipe,
_In_ SystemPipeSegmentBuffer_t* Buffer,
_In_ uint8_t* Data,
_In_ size_t Length)
{
// Variables
unsigned int ReadIndex;
unsigned int WriteIndex;
size_t BytesAvailable = 0;
size_t BytesRead = 0;
size_t BytesCommitted;
// Make sure there are bytes to read
while (BytesRead < Length) {
// Use the write-comitted
WriteIndex = atomic_load(&Buffer->WriteCommitted);
ReadIndex = atomic_load(&Buffer->ReadPointer);
BytesAvailable = MIN(
CalculateBytesAvailableForReading(Buffer, ReadIndex, WriteIndex),
Length - BytesRead);
BytesCommitted = BytesAvailable;
if (!BytesAvailable) {
if (Pipe->Configuration & PIPE_NOBLOCK) {
break;
}
SchedulerAtomicThreadSleep((atomic_int*)&Buffer->WriteCommitted, (int*)&WriteIndex, 0);
continue; // Start over
}
// Synchronize with other consumers
if (Pipe->Configuration & PIPE_MULTIPLE_CONSUMERS) {
while (BytesAvailable) {
size_t NewReadPointer = ReadIndex + BytesAvailable;
if (atomic_compare_exchange_weak(&Buffer->ReadPointer, &ReadIndex, NewReadPointer)) {
break;
}
WriteIndex = atomic_load(&Buffer->WriteCommitted);
BytesAvailable = MIN(
CalculateBytesAvailableForReading(Buffer, ReadIndex, WriteIndex),
Length - BytesRead);
}
if (!BytesAvailable) {
continue; // Start over as we ran out
}
// Wait for our turn
BytesCommitted = atomic_load(&Buffer->ReadCommitted);
while (BytesCommitted < ReadIndex) {
BytesCommitted = atomic_load(&Buffer->ReadCommitted);
}
BytesCommitted = BytesAvailable;
}
else {
atomic_store_explicit(&Buffer->ReadPointer, ReadIndex + BytesAvailable, memory_order_relaxed);
}
// Write the data to the provided buffer
while (BytesAvailable--) {
Data[BytesRead++] = Buffer->Pointer[(ReadIndex++ & (Buffer->Size - 1))];
}
atomic_fetch_add(&Buffer->ReadCommitted, BytesCommitted);
SchedulerHandleSignal((uintptr_t*)&Buffer->ReadCommitted);
// If it was possible read bytes, return. With raw bytes we allow
// the reader to read less, however never allow to read 0
if (BytesRead > 0) {
break;
}
}
return BytesRead;
}
/////////////////////////////////////////////////////////////////////////
// System Pipe Raw Buffer Code
static size_t
WriteRawSegmentBuffer(
_In_ SystemPipe_t* Pipe,
_In_ SystemPipeSegmentBuffer_t* Buffer,
_In_ const uint8_t* Data,
_In_ size_t Length)
{
// Variables
unsigned int ReadIndex;
unsigned int WriteIndex;
size_t BytesWritten = 0;
size_t BytesAvailable;
size_t BytesCommitted;
// Make sure we write all the bytes
while (BytesWritten < Length) {
WriteIndex = atomic_load(&Buffer->WritePointer);
ReadIndex = atomic_load(&Buffer->ReadCommitted);
BytesAvailable = MIN(
CalculateBytesAvailableForWriting(Buffer, ReadIndex, WriteIndex),
Length - BytesWritten);
BytesCommitted = BytesAvailable;
if (!BytesAvailable) {
if (Pipe->Configuration & PIPE_NOBLOCK) {
break;
}
SchedulerAtomicThreadSleep((atomic_int*)&Buffer->ReadCommitted, (int*)&ReadIndex, 0);
continue; // Start over
}
// Synchronize with other producers
if (Pipe->Configuration & PIPE_MULTIPLE_PRODUCERS) {
while (BytesAvailable) {
size_t NewWritePointer = WriteIndex + BytesAvailable;
if (atomic_compare_exchange_weak(&Buffer->WritePointer, &WriteIndex, NewWritePointer)) {
break;
}
ReadIndex = atomic_load(&Buffer->ReadCommitted);
BytesAvailable = MIN(
CalculateBytesAvailableForWriting(Buffer, ReadIndex, WriteIndex),
Length - BytesWritten);
}
if (!BytesAvailable) {
continue; // Start over as we ran out
}
// Wait for our turn
BytesCommitted = atomic_load(&Buffer->WriteCommitted);
while (BytesCommitted < WriteIndex) {
BytesCommitted = atomic_load(&Buffer->WriteCommitted);
}
BytesCommitted = BytesAvailable;
}
else {
atomic_store_explicit(&Buffer->WritePointer, WriteIndex + BytesAvailable, memory_order_relaxed);
}
// Write the data to the internal buffer
while (BytesAvailable--) {
Buffer->Pointer[(WriteIndex++ & (Buffer->Size - 1))] = Data[BytesWritten++];
}
atomic_fetch_add(&Buffer->WriteCommitted, BytesCommitted);
SchedulerHandleSignal((uintptr_t*)&Buffer->WriteCommitted);
}
return BytesWritten;
}