/* * Allocate new node * If freeNodes list empty and # nodes < maxNodes, allocate new node */ stpcNode* _newNode(stpcProxy* proxy, bool alloc) { stpcNode* node = NULL; sequencedPtr oldFree, newFree; oldFree.ival = atomic_load_explicit(&proxy->freeHead.ival, memory_order_acquire); while (oldFree.ptr != atomic_load_explicit(&proxy->freeTail, memory_order_relaxed)) { newFree.ptr = oldFree.ptr->next; newFree.sequence = oldFree.sequence + 1; if (atomic_compare_exchange_strong_explicit(&proxy->freeHead.ival, &oldFree.ival, newFree.ival, memory_order_acq_rel, memory_order_acquire)) { node = oldFree.ptr; stpcGetLocalStats(proxy)->reuse++; break; } } if (node == NULL && alloc) { int oldNum = atomic_load_explicit(&proxy->numNodes, memory_order_relaxed); while (oldNum < proxy->maxNodes && !atomic_compare_exchange_strong_explicit(&proxy->numNodes, &oldNum, oldNum + 1, memory_order_relaxed, memory_order_relaxed)); if (oldNum >= proxy->maxNodes) return NULL; node = proxy->allocMem(sizeof(stpcNode)); if (node == NULL) return NULL; } memset(node, 0, sizeof(stpcNode)); return node; }
void exchange (atomic_int *i) { int r; atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_seq_cst, memory_order_release); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_seq_cst, memory_order_acq_rel); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_relaxed, memory_order_consume); /* { dg-warning "failure memory model cannot be stronger" } */ atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_seq_cst, memory_order_release); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_seq_cst, memory_order_acq_rel); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_relaxed, memory_order_consume); /* { dg-warning "failure memory model cannot be stronger" } */ }
void enqueue(queue_t *q, unsigned int val) { std::string str1("enqueue"); //ANNOTATION function_call(str1, INVOCATION, val); //ANNOTATION int success = 0; unsigned int node; pointer tail; pointer next; pointer tmp; node = new_node(); store_32(&q->nodes[node].value, val); tmp = atomic_load_explicit(&q->nodes[node].next, memory_order_seq_cst); set_ptr(&tmp, 0); // NULL atomic_store_explicit(&q->nodes[node].next, tmp, memory_order_seq_cst); while (!success) { tail = atomic_load_explicit(&q->tail, memory_order_seq_cst); next = atomic_load_explicit(&q->nodes[get_ptr(tail)].next, memory_order_seq_cst); if (tail == atomic_load_explicit(&q->tail, memory_order_seq_cst)) { /* Check for uninitialized 'next' */ MODEL_ASSERT(get_ptr(next) != POISON_IDX); if (get_ptr(next) == 0) { // == NULL pointer value = MAKE_POINTER(node, get_count(next) + 1); success = atomic_compare_exchange_strong_explicit(&q->nodes[get_ptr(tail)].next, &next, value, memory_order_seq_cst, memory_order_seq_cst); } if (!success) { unsigned int ptr = get_ptr(atomic_load_explicit(&q->nodes[get_ptr(tail)].next, memory_order_seq_cst)); pointer value = MAKE_POINTER(ptr, get_count(tail) + 1); atomic_compare_exchange_strong_explicit(&q->tail, &tail, value, memory_order_seq_cst, memory_order_seq_cst); thrd_yield(); } } } atomic_compare_exchange_strong_explicit(&q->tail, &tail, MAKE_POINTER(node, get_count(tail) + 1), memory_order_seq_cst, memory_order_seq_cst); function_call(str1, RESPONSE); //ANNOTATION }
void _queueNode(stpcProxy* proxy, stpcNode* newNode) { sequencedPtr oldTail; sequencedPtr newTail; stpcNode *tailNode, *next; bool rc, rc2; long attempts = 0; newNode->count = GUARD_BIT + 2 * REFERENCE; /* * monkey through the trees queuing trick */ newTail.ptr = newNode; newTail.sequence = 0; oldTail.ival = atomic_load_explicit(&proxy->tail.ival, memory_order_consume); do { attempts++; } while (!atomic_compare_exchange_strong_explicit(&proxy->tail.ival, &oldTail.ival, newTail.ival, memory_order_acq_rel, memory_order_acquire)); atomic_store_explicit(&oldTail.ptr->next, newNode, memory_order_relaxed); // update old node's reference count by number of acquired references, clear guard bit, and drop ref acquired from tail pointer _dropProxyNodeReference(proxy, oldTail.ptr, (oldTail.sequence - GUARD_BIT)); stats_t *stats = stpcGetLocalStats(proxy); stats->tries++; // _addNode invocations stats->attempts += attempts; // tail enqueue attempts }
/* Test for consistency on sizes 1, 2, 4, 8, 16 and 32. */ int main () { test_struct c; atomic_store_explicit (&a, zero, memory_order_relaxed); if (memcmp (&a, &zero, size)) abort (); c = atomic_exchange_explicit (&a, ones, memory_order_seq_cst); if (memcmp (&c, &zero, size)) abort (); if (memcmp (&a, &ones, size)) abort (); b = atomic_load_explicit (&a, memory_order_relaxed); if (memcmp (&b, &ones, size)) abort (); if (!atomic_compare_exchange_strong_explicit (&a, &b, zero, memory_order_seq_cst, memory_order_acquire)) abort (); if (memcmp (&a, &zero, size)) abort (); if (atomic_compare_exchange_weak_explicit (&a, &b, ones, memory_order_seq_cst, memory_order_acquire)) abort (); if (memcmp (&b, &zero, size)) abort (); return 0; }
static noreturn void thread_entry(cloudabi_tid_t tid, void *data) { // Set up TLS space. char tls_space[tls_size()]; char *tls_start = tls_addr(tls_space); memcpy(tls_start, __pt_tls_vaddr_abs, __pt_tls_filesz); memset(tls_start + __pt_tls_filesz, '\0', __pt_tls_memsz_aligned - __pt_tls_filesz); tls_replace(tls_space); // Fix up some of the variables stored in TLS. pthread_t handle = data; __pthread_self_object = handle; __pthread_thread_id = tid; __safestack_unsafe_stack_ptr = (void *)__rounddown( (uintptr_t)handle->unsafe_stack + handle->unsafe_stacksize, PTHREAD_UNSAFE_STACK_ALIGNMENT); // Initialize the the once object used for joining. This is also done // by the parent, but it may be the case that we call pthread_exit() // before the parent has a chance to initialize it. cloudabi_lock_t expected = CLOUDABI_LOCK_BOGUS; atomic_compare_exchange_strong_explicit( &handle->join, &expected, tid | CLOUDABI_LOCK_WRLOCKED, memory_order_acquire, memory_order_relaxed); // Free thread startup information and invoke user-supplied start routine. void *(*start_routine)(void *) = handle->start_routine; void *argument = handle->argument; pthread_exit(start_routine(argument)); }
int take(Deque *q) { std::string str1("pop_back"); //ANNOTATION function_call(str1, INVOCATION); //ANNOTATION size_t b = atomic_load_explicit(&q->bottom, memory_order_seq_cst) - 1; Array *a = (Array *) atomic_load_explicit(&q->array, memory_order_seq_cst); atomic_store_explicit(&q->bottom, b, memory_order_seq_cst); //relaxed atomic_thread_fence(memory_order_seq_cst); size_t t = atomic_load_explicit(&q->top, memory_order_seq_cst); int x; if (t <= b) { /* Non-empty queue. */ x = atomic_load_explicit(&a->buffer[b % atomic_load_explicit(&a->size,memory_order_seq_cst)], memory_order_seq_cst); if (t == b) { /* Single last element in queue. */ if (!atomic_compare_exchange_strong_explicit(&q->top, &t, t + 1, memory_order_seq_cst, memory_order_seq_cst)) /* Failed race. */ x = EMPTY; atomic_store_explicit(&q->bottom, b + 1, memory_order_seq_cst); //relaxed } } else { /* Empty queue. */ x = EMPTY; atomic_store_explicit(&q->bottom, b + 1, memory_order_seq_cst); // relaxed } //if(x == EMPTY) //function_call(str1, RESPONSE, (uint64_t) NULL); //ANNOTATION //else function_call(str1, RESPONSE, (uint64_t) x); //ANNOTATION return x; }
bool dequeue(queue_t *q, unsigned int *retVal) { std::string str1("dequeue"); //ANNOTATION function_call(str1, INVOCATION); //ANNOTATION int success = 0; pointer head; pointer tail; pointer next; while (!success) { head = atomic_load_explicit(&q->head, memory_order_seq_cst); tail = atomic_load_explicit(&q->tail, memory_order_seq_cst); next = atomic_load_explicit(&q->nodes[get_ptr(head)].next, memory_order_seq_cst); if (atomic_load_explicit(&q->head, memory_order_seq_cst) == head) { if (get_ptr(head) == get_ptr(tail)) { /* Check for uninitialized 'next' */ MODEL_ASSERT(get_ptr(next) != POISON_IDX); if (get_ptr(next) == 0) { // NULL function_call(str1, RESPONSE); //ANNOTATION return false; // NULL } atomic_compare_exchange_strong_explicit(&q->tail, &tail, MAKE_POINTER(get_ptr(next), get_count(tail) + 1), memory_order_seq_cst, memory_order_seq_cst); thrd_yield(); } else { *retVal = load_32(&q->nodes[get_ptr(next)].value); success = atomic_compare_exchange_strong_explicit(&q->head, &head, MAKE_POINTER(get_ptr(next), get_count(head) + 1), memory_order_seq_cst, memory_order_seq_cst); if (!success) thrd_yield(); } } } reclaim(get_ptr(head)); function_call(str1, RESPONSE, *retVal); //ANNOTATION return true; }
static _Bool SetNextSegment( _In_ SystemPipeSegment_t* Segment, _In_ SystemPipeSegment_t* Next) { SystemPipeSegment_t* Expected = NULL; return atomic_compare_exchange_strong_explicit(&Segment->Link, &Expected, Next, memory_order_release, memory_order_relaxed); }
static inline _Bool SwapSystemPipeHead( _In_ SystemPipe_t* Pipe, _In_ SystemPipeSegment_t** Segment, _In_ SystemPipeSegment_t* Next) { assert(Pipe->Configuration & PIPE_MULTIPLE_CONSUMERS); return atomic_compare_exchange_strong_explicit(&Pipe->ConsumerState.Head, Segment, Next, memory_order_release, memory_order_acquire); }
static inline _Bool SwapSystemPipeTail( _In_ SystemPipe_t* Pipe, _In_ SystemPipeSegment_t** Segment, _In_ SystemPipeSegment_t* Next) { assert((Pipe->Configuration & PIPE_MPMC) != 0); return atomic_compare_exchange_strong_explicit(&Pipe->ProducerState.Tail, Segment, Next, memory_order_release, memory_order_relaxed); }
TEST(stdatomic, atomic_compare_exchange) { atomic_int i; int expected; atomic_store(&i, 123); expected = 123; ASSERT_TRUE(atomic_compare_exchange_strong(&i, &expected, 456)); ASSERT_FALSE(atomic_compare_exchange_strong(&i, &expected, 456)); ASSERT_EQ(456, expected); atomic_store(&i, 123); expected = 123; ASSERT_TRUE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_FALSE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_EQ(456, expected); atomic_store(&i, 123); expected = 123; int iter_count = 0; do { ++iter_count; ASSERT_LT(iter_count, 100); // Arbitrary limit on spurious compare_exchange failures. ASSERT_EQ(expected, 123); } while(!atomic_compare_exchange_weak(&i, &expected, 456)); ASSERT_FALSE(atomic_compare_exchange_weak(&i, &expected, 456)); ASSERT_EQ(456, expected); atomic_store(&i, 123); expected = 123; iter_count = 0; do { ++iter_count; ASSERT_LT(iter_count, 100); ASSERT_EQ(expected, 123); } while(!atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_FALSE(atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_EQ(456, expected); }
int pthread_mutex_destroy(pthread_mutex_t* mutex_interface) { pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); // Store 0xffff to make the mutex unusable. Although POSIX standard says it is undefined // behavior to destroy a locked mutex, we prefer not to change mutex->state in that situation. if (MUTEX_STATE_BITS_IS_UNLOCKED(old_state) && atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 0xffff, memory_order_relaxed, memory_order_relaxed)) { return 0; } return EBUSY; }
static inline __always_inline int __pthread_normal_mutex_trylock(pthread_mutex_internal_t* mutex, uint16_t shared) { const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; const uint16_t locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; uint16_t old_state = unlocked; if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, locked_uncontended, memory_order_acquire, memory_order_relaxed))) { return 0; } return EBUSY; }
/** * Unset thread specific value if present * * @return true on success */ bool erase() { auto currThread = current_thread(); for (size_t i = 0; i < N; i++) { if (atomic_load_explicit(&threads[i], memory_order_acquire) == currThread) { values[i] = {}; thread_t nullThread = nullptr; return atomic_compare_exchange_strong_explicit(&threads[i], &currThread, nullThread, memory_order_acq_rel, memory_order_acq_rel); } } return false; }
stpcNode* stpcGetProxyNodeReference(stpcProxy* proxy) { sequencedPtr oldTail; sequencedPtr newTail; oldTail.ival = atomic_load_explicit(&proxy->tail.ival, memory_order_relaxed); do { newTail.sequence = oldTail.sequence + REFERENCE; newTail.ptr = oldTail.ptr; } while (!atomic_compare_exchange_strong_explicit(&proxy->tail.ival, &oldTail.ival, newTail.ival, memory_order_relaxed, memory_order_relaxed)); return oldTail.ptr; }
void pony_asio_event_subscribe(asio_event_t* ev) { if((ev == NULL) || (ev->flags == ASIO_DISPOSABLE) || (ev->flags == ASIO_DESTROYED)) return; asio_backend_t* b = ponyint_asio_get_backend(); if(ev->noisy) ponyint_asio_noisy_add(); struct epoll_event ep; ep.data.ptr = ev; ep.events = EPOLLRDHUP | EPOLLET; if(ev->flags & ASIO_READ) ep.events |= EPOLLIN; if(ev->flags & ASIO_WRITE) ep.events |= EPOLLOUT; if(ev->flags & ASIO_TIMER) { ev->fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); timer_set_nsec(ev->fd, ev->nsec); ep.events |= EPOLLIN; } if(ev->flags & ASIO_SIGNAL) { int sig = (int)ev->nsec; asio_event_t* prev = NULL; if((sig < MAX_SIGNAL) && atomic_compare_exchange_strong_explicit(&b->sighandlers[sig], &prev, ev, memory_order_release, memory_order_relaxed)) { signal(sig, signal_handler); ev->fd = eventfd(0, EFD_NONBLOCK); ep.events |= EPOLLIN; } else { return; } } epoll_ctl(b->epfd, EPOLL_CTL_ADD, ev->fd, &ep); }
bool ponyint_messageq_markempty(messageq_t* q) { pony_msg_t* tail = q->tail; pony_msg_t* head = atomic_load_explicit(&q->head, memory_order_relaxed); if(((uintptr_t)head & 1) != 0) return true; if(head != tail) return false; head = (pony_msg_t*)((uintptr_t)head | 1); return atomic_compare_exchange_strong_explicit(&q->head, &tail, head, memory_order_release, memory_order_relaxed); }
void pony_asio_event_unsubscribe(asio_event_t* ev) { if((ev == NULL) || (ev->flags == ASIO_DISPOSABLE) || (ev->flags == ASIO_DESTROYED)) return; asio_backend_t* b = ponyint_asio_get_backend(); if(ev->noisy) { ponyint_asio_noisy_remove(); ev->noisy = false; } epoll_ctl(b->epfd, EPOLL_CTL_DEL, ev->fd, NULL); if(ev->flags & ASIO_TIMER) { if(ev->fd != -1) { close(ev->fd); ev->fd = -1; } } if(ev->flags & ASIO_SIGNAL) { int sig = (int)ev->nsec; asio_event_t* prev = ev; if((sig < MAX_SIGNAL) && atomic_compare_exchange_strong_explicit(&b->sighandlers[sig], &prev, NULL, memory_order_release, memory_order_relaxed)) { signal(sig, SIG_DFL); close(ev->fd); ev->fd = -1; } } ev->flags = ASIO_DISPOSABLE; send_request(ev, ASIO_DISPOSABLE); }
bool ponyint_messageq_markempty(messageq_t* q) { pony_msg_t* tail = q->tail; pony_msg_t* head = atomic_load_explicit(&q->head, memory_order_relaxed); if(((uintptr_t)head & 1) != 0) return true; if(head != tail) return false; head = (pony_msg_t*)((uintptr_t)head | 1); #ifdef USE_VALGRIND ANNOTATE_HAPPENS_BEFORE(&q->head); #endif return atomic_compare_exchange_strong_explicit(&q->head, &tail, head, memory_order_release, memory_order_relaxed); }
int pthread_mutex_trylock(pthread_mutex_t* mutex_interface) { pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); uint16_t mtype = (old_state & MUTEX_TYPE_MASK); uint16_t shared = (old_state & MUTEX_SHARED_MASK); const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; // Handle common case first. if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) { return __pthread_normal_mutex_trylock(mutex, shared); } // Do we already own this recursive or error-check mutex? pid_t tid = __get_thread()->tid; if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { return EBUSY; } return __recursive_increment(mutex, old_state); } // Same as pthread_mutex_lock, except that we don't want to wait, and // the only operation that can succeed is a single compare_exchange to acquire the // lock if it is released / not owned by anyone. No need for a complex loop. // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. old_state = unlocked; if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, locked_uncontended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } return EBUSY; }
/** * Set or overwrite thread specific value * * @param value value to store * * @return true on success */ bool set(T value) { auto currThread = current_thread(); T *ptr = nullptr; // Find previous value if any for (size_t i = 0; ptr == nullptr && i < N; i++) if (atomic_load_explicit(&threads[i], memory_order_acquire) == currThread) ptr = &values[i]; // Find null value if any for (size_t i = 0; ptr == nullptr && i < N; i++) { thread_t nullThread = nullptr; if (atomic_compare_exchange_strong_explicit(&threads[i], &nullThread, currThread, memory_order_acq_rel, memory_order_acq_rel)) ptr = &values[i]; } // Insert if we can if (ptr) *ptr = value; return ptr != nullptr; }
/** * Update speed meter. * @param[in] speed * @param[in] count */ void spdm_update(struct speed_meter *speed, uint64_t count) { uint64_t curr_time = zclock(false); uint64_t last_update = atomic_load_explicit(&speed->last_update, memory_order_acquire); if (curr_time - last_update >= SEC2USEC(1)) { if (atomic_compare_exchange_strong_explicit(&speed->last_update, &last_update, curr_time, memory_order_release, memory_order_relaxed)) { size_t i = atomic_load_explicit(&speed->i, memory_order_acquire); uint64_t speed_aux = atomic_load_explicit(&speed->speed_aux, memory_order_acquire); atomic_store_explicit(&speed->backlog[i].speed, speed_aux, memory_order_release); atomic_fetch_sub_explicit(&speed->speed_aux, speed_aux, memory_order_release); atomic_store_explicit(&speed->backlog[i].timestamp, last_update, memory_order_release); i++; if (SPEED_METER_BACKLOG == i) { i = 0; } atomic_store_explicit(&speed->i, i, memory_order_release); } } atomic_fetch_add_explicit(&speed->speed_aux, count, memory_order_release); }
void BBinder::attachObject( const void* objectID, void* object, void* cleanupCookie, object_cleanup_func func) { Extras* e = reinterpret_cast<Extras*>( atomic_load_explicit(&mExtras, std::memory_order_acquire)); if (!e) { e = new Extras; uintptr_t expected = 0; if (!atomic_compare_exchange_strong_explicit( &mExtras, &expected, reinterpret_cast<uintptr_t>(e), std::memory_order_release, std::memory_order_acquire)) { delete e; e = reinterpret_cast<Extras*>(expected); // Filled in by CAS } if (e == 0) return; // out of memory } AutoMutex _l(e->mLock); e->mObjects.attach(objectID, object, cleanupCookie, func); }
int steal(Deque *q) { std::string str1("pop_front"); //ANNOTATION function_call(str1, INVOCATION); //ANNOTATION size_t t = atomic_load_explicit(&q->top, memory_order_seq_cst); //acquire atomic_thread_fence(memory_order_seq_cst); size_t b = atomic_load_explicit(&q->bottom, memory_order_seq_cst); //acquire int x = EMPTY; if (t < b) { /* Non-empty queue. */ Array *a = (Array *) atomic_load_explicit(&q->array, memory_order_seq_cst); x = atomic_load_explicit(&a->buffer[t % atomic_load_explicit(&a->size, memory_order_seq_cst)], memory_order_seq_cst); if (!atomic_compare_exchange_strong_explicit(&q->top, &t, t + 1, memory_order_seq_cst, memory_order_seq_cst)) { function_call(str1, RESPONSE, (uint64_t) ABORT); //ANNOTATION /* Failed race. */ return ABORT; } } //if(x == EMPTY) //function_call(str1, RESPONSE, (uint64_t) NULL); //ANNOTATION //else function_call(str1, RESPONSE, (uint64_t) x); //ANNOTATION return x; }
static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex, bool use_realtime_clock, const timespec* abs_timeout_or_null) { uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); uint16_t mtype = (old_state & MUTEX_TYPE_MASK); uint16_t shared = (old_state & MUTEX_SHARED_MASK); // Handle common case first. if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) { return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null); } // Do we already own this recursive or error-check mutex? pid_t tid = __get_thread()->tid; if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { return EDEADLK; } return __recursive_increment(mutex, old_state); } const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; // First, if the mutex is unlocked, try to quickly acquire it. // In the optimistic case where this works, set the state to locked_uncontended. if (old_state == unlocked) { // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, locked_uncontended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } } ScopedTrace trace("Contending for pthread mutex"); while (true) { if (old_state == unlocked) { // NOTE: We put the state to locked_contended since we _know_ there // is contention when we are in this loop. This ensures all waiters // will be unlocked. // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state, &old_state, locked_contended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } continue; } else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) { // We should set it to locked_contended beforing going to sleep. This can make // sure waiters will be woken up eventually. int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state); if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state, &old_state, new_state, memory_order_relaxed, memory_order_relaxed))) { continue; } old_state = new_state; } int result = check_timespec(abs_timeout_or_null, true); if (result != 0) { return result; } // We are in locked_contended state, sleep until someone wakes us up. if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock, abs_timeout_or_null) == -ETIMEDOUT) { return ETIMEDOUT; } old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); } }
PONY_API void pony_asio_event_unsubscribe(asio_event_t* ev) { if((ev == NULL) || (ev->flags == ASIO_DISPOSABLE) || (ev->flags == ASIO_DESTROYED)) { pony_assert(0); return; } asio_backend_t* b = ponyint_asio_get_backend(); pony_assert(b != NULL); if(ev->noisy) { uint64_t old_count = ponyint_asio_noisy_remove(); // tell scheduler threads that asio has no noisy actors // if the old_count was 1 if (old_count == 1) { ponyint_sched_unnoisy_asio(SPECIAL_THREADID_EPOLL); // maybe wake up a scheduler thread if they've all fallen asleep ponyint_sched_maybe_wakeup_if_all_asleep(-1); } ev->noisy = false; } epoll_ctl(b->epfd, EPOLL_CTL_DEL, ev->fd, NULL); if(ev->flags & ASIO_TIMER) { if(ev->fd != -1) { close(ev->fd); ev->fd = -1; } } if(ev->flags & ASIO_SIGNAL) { int sig = (int)ev->nsec; asio_event_t* prev = ev; #ifdef USE_VALGRIND ANNOTATE_HAPPENS_BEFORE(&b->sighandlers[sig]); #endif if((sig < MAX_SIGNAL) && atomic_compare_exchange_strong_explicit(&b->sighandlers[sig], &prev, NULL, memory_order_release, memory_order_relaxed)) { struct sigaction new_action; #if !defined(USE_SCHEDULER_SCALING_PTHREADS) // Make sure we ignore signals related to scheduler sleeping/waking // as the default for those signals is termination if(sig == PONY_SCHED_SLEEP_WAKE_SIGNAL) new_action.sa_handler = empty_signal_handler; else #endif new_action.sa_handler = SIG_DFL; sigemptyset (&new_action.sa_mask); // ask to restart interrupted syscalls to match `signal` behavior new_action.sa_flags = SA_RESTART; sigaction(sig, &new_action, NULL); close(ev->fd); ev->fd = -1; } } ev->flags = ASIO_DISPOSABLE; send_request(ev, ASIO_DISPOSABLE); }
PONY_API void pony_asio_event_subscribe(asio_event_t* ev) { if((ev == NULL) || (ev->flags == ASIO_DISPOSABLE) || (ev->flags == ASIO_DESTROYED)) { pony_assert(0); return; } asio_backend_t* b = ponyint_asio_get_backend(); pony_assert(b != NULL); if(ev->noisy) { uint64_t old_count = ponyint_asio_noisy_add(); // tell scheduler threads that asio has at least one noisy actor // if the old_count was 0 if (old_count == 0) ponyint_sched_noisy_asio(SPECIAL_THREADID_EPOLL); } struct epoll_event ep; ep.data.ptr = ev; ep.events = EPOLLRDHUP; if(ev->flags & ASIO_READ) ep.events |= EPOLLIN; if(ev->flags & ASIO_WRITE) ep.events |= EPOLLOUT; if(ev->flags & ASIO_TIMER) { ev->fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK); timer_set_nsec(ev->fd, ev->nsec); ep.events |= EPOLLIN; } if(ev->flags & ASIO_SIGNAL) { int sig = (int)ev->nsec; asio_event_t* prev = NULL; #ifdef USE_VALGRIND ANNOTATE_HAPPENS_BEFORE(&b->sighandlers[sig]); #endif if((sig < MAX_SIGNAL) && atomic_compare_exchange_strong_explicit(&b->sighandlers[sig], &prev, ev, memory_order_release, memory_order_relaxed)) { struct sigaction new_action; new_action.sa_handler = signal_handler; sigemptyset (&new_action.sa_mask); // ask to restart interrupted syscalls to match `signal` behavior new_action.sa_flags = SA_RESTART; sigaction(sig, &new_action, NULL); ev->fd = eventfd(0, EFD_NONBLOCK); ep.events |= EPOLLIN; } else { return; } } if(ev->flags & ASIO_ONESHOT) { ep.events |= EPOLLONESHOT; } else { // Only use edge triggering if one shot isn't enabled. // This is because of how the runtime gets notifications // from epoll in this ASIO thread and then notifies the // appropriate actor to read/write as necessary. // specifically, it seems there's an edge case/race condition // with edge triggering where if there is already data waiting // on the socket, then epoll might not be triggering immediately // when an edge triggered epoll request is made. ep.events |= EPOLLET; } epoll_ctl(b->epfd, EPOLL_CTL_ADD, ev->fd, &ep); }